<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  
  <meta name="renderer" content="webkit">
  <meta http-equiv="X-UA-Compatible" content="IE=edge" >
  <link rel="dns-prefetch" href="http://cxsw_zyj.gitee.io">
  <title>Linux_宸轩少翁</title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta property="og:type" content="website">
<meta property="og:title" content="Linux_宸轩少翁">
<meta property="og:url" content="http://cxsw_zyj.gitee.io/page/2/index.html">
<meta property="og:site_name" content="Linux_宸轩少翁">
<meta property="og:locale" content="en_US">
<meta property="article:author" content="宸轩少翁">
<meta name="twitter:card" content="summary">
  
    <link rel="alternative" href="/atom.xml" title="Linux_宸轩少翁" type="application/atom+xml">
  
  
    <link rel="icon" href="/assets/my_favicon.ico">
  
  <link rel="stylesheet" type="text/css" href="/./main.0cf68a.css">
  <style type="text/css">
  
    #container.show {
      background: linear-gradient(200deg,#a0cfe4,#e8c37e);
    }
  </style>
  

  

<meta name="generator" content="Hexo 5.2.0"></head>

<body>
  <div id="container" q-class="show:isCtnShow">
    <canvas id="anm-canvas" class="anm-canvas"></canvas>
    <div class="left-col" q-class="show:isShow">
      
<div class="overlay" style="background: #4d4d4d"></div>
<div class="intrude-less" style="width:100%;">
	<header id="header" class="inner">
		<a href="/" class="profilepic">
			<img src="/assets/1.jpg" class="js-avatar">
		</a>
		<hgroup>
		  <h1 class="header-author"><a href="/"></a></h1>
		</hgroup>
		

		<nav class="header-menu">
			<ul>
			
				<li><a href="/">主页</a></li>
	        
				<li><a href="/categories">分类</a></li>
	        
				<li><a href="/categories/%E6%9D%82%E6%96%87%E9%87%8E%E8%AF%97">杂文野诗</a></li>
	        
			</ul>
		</nav>
		<nav class="header-smart-menu">
    		
    			
    			<a q-on="click: openSlider(e, 'innerArchive')" href="javascript:void(0)">所有文章</a>
    			
            
    			
    			<a q-on="click: openSlider(e, 'friends')" href="javascript:void(0)">友链</a>
    			
            
    			
    			<a q-on="click: openSlider(e, 'aboutme')" href="javascript:void(0)">关于我</a>
    			
            
		</nav>
		<nav class="header-nav">
			<div class="social">
				
					<a class="github" target="_blank" href="#" title="github"><i class="icon-github"></i></a>
		        
					<a class="weibo" target="_blank" href="#" title="weibo"><i class="icon-weibo"></i></a>
		        
					<a class="rss" target="_blank" href="#" title="rss"><i class="icon-rss"></i></a>
		        
					<a class="zhihu" target="_blank" href="#" title="zhihu"><i class="icon-zhihu"></i></a>
		        
			</div>
		</nav>

<div style="position:absolute;left:20px;width:85%">
	<iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=260 height=86 src="//music.163.com/outchain/player?type=2&id=1325711950&auto=0&height=66"></iframe>
	</div>
	</header>		
</div>

    </div>
    <div class="mid-col" q-class="show:isShow,hide:isShow|isFalse">
      
<nav id="mobile-nav">
  	<div class="overlay js-overlay" style="background: #4d4d4d"></div>
	<div class="btnctn js-mobile-btnctn">
  		<div class="slider-trigger list" q-on="click: openSlider(e)"><i class="icon icon-sort"></i></div>
	</div>
	<div class="intrude-less">
		<header id="header" class="inner">
			<div class="profilepic">
				<img src="/assets/1.jpg" class="js-avatar">
			</div>
			<hgroup>
			  <h1 class="header-author js-header-author"></h1>
			</hgroup>
			
			
			
				
			
				
			
				
			
			
			
			<nav class="header-nav">
				<div class="social">
					
						<a class="github" target="_blank" href="#" title="github"><i class="icon-github"></i></a>
			        
						<a class="weibo" target="_blank" href="#" title="weibo"><i class="icon-weibo"></i></a>
			        
						<a class="rss" target="_blank" href="#" title="rss"><i class="icon-rss"></i></a>
			        
						<a class="zhihu" target="_blank" href="#" title="zhihu"><i class="icon-zhihu"></i></a>
			        
				</div>
			</nav>

			<nav class="header-menu js-header-menu">
				<ul style="width: 70%">
				
				
					<li style="width: 33.333333333333336%"><a href="/">主页</a></li>
		        
					<li style="width: 33.333333333333336%"><a href="/categories">分类</a></li>
		        
					<li style="width: 33.333333333333336%"><a href="/categories/%E6%9D%82%E6%96%87%E9%87%8E%E8%AF%97">杂文野诗</a></li>
		        
				</ul>
			</nav>
		</header>				
	</div>
	<div class="mobile-mask" style="display:none" q-show="isShow"></div>
</nav>

      <div id="wrapper" class="body-wrap">
        <div class="menu-l">
          <div class="canvas-wrap">
            <canvas data-colors="#eaeaea" data-sectionHeight="100" data-contentId="js-content" id="myCanvas1" class="anm-canvas"></canvas>
          </div>
          <div id="js-content" class="content-ll">
            
  
    <article id="post-kubernetes/k8s资源调度RC&amp;RS&amp;Deployment" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6RC&RS&Deployment/">k8s资源调度RC&amp;RS&amp;Deployment</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6RC&RS&Deployment/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="1-1-Replication-Controller和ReplicaSet"><a href="#1-1-Replication-Controller和ReplicaSet" class="headerlink" title="1.1  Replication Controller和ReplicaSet"></a>1.1  Replication Controller和ReplicaSet</h2><p>Replication Controller（复制控制器，RC）和ReplicaSet（复制集，RS）是两种简单部署Pod的方式。因为在生产环境中，<strong>主要使用更高级的Deployment等方式进行Pod的管理和部署，所以本节只对Replication Controller和Replica Set的部署方式进行简单介绍。</strong></p>
<h3 id="1-1-1-Replication-Controller（不常用，且已被弃用）"><a href="#1-1-1-Replication-Controller（不常用，且已被弃用）" class="headerlink" title="1.1.1  Replication Controller（不常用，且已被弃用）"></a>1.1.1  Replication Controller（不常用，且已被弃用）</h3><p>Replication Controller（简称RC）可确保Pod副本数达到期望值，也就是RC定义的数量。换句话说，Replication Controller可确保一个Pod或一组同类Pod总是可用。</p>
<p>如果存在的Pod大于设定的值，则Replication Controller将终止额外的Pod。</p>
<p>如果太小，Replication Controller将启动更多的Pod用于保证达到期望值。与</p>
<p>手动创建Pod不同的是，用Replication Controller维护的Pod在失败、删除或终止时会自动替换。</p>
<p>因此即使应用程序只需要一个Pod，也应该使用Replication Controller或其他方式管理。</p>
<p>Replication Controller类似于进程管理程序，但是Replication Controller不是监视单个节点上的各个进程，而是监视多个节点上的多个Pod。</p>
<h4 id="1-1-1-1-Replication-Controller-示例"><a href="#1-1-1-1-Replication-Controller-示例" class="headerlink" title="1.1.1.1 Replication Controller 示例"></a>1.1.1.1 Replication Controller 示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ReplicationController</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">3</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">image:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">ports:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">containerPort:</span> <span class="number">80</span></span><br></pre></td></tr></table></figure>





<h3 id="1-1-2-ReplicaSet（不常单独使用）"><a href="#1-1-2-ReplicaSet（不常单独使用）" class="headerlink" title="1.1.2  ReplicaSet（不常单独使用）"></a>1.1.2  ReplicaSet（不常单独使用）</h3><p>ReplicaSet是支持基于集合的标签选择器的下一代Replication Controller，它主要用作Deployment协调创建、删除和更新Pod，和Replication Controller唯一的区别是，ReplicaSet支持标签选择器。在实际应用中，虽然ReplicaSet可以单独使用，但是一般建议使用Deployment来自动管理ReplicaSet，除非自定义的Pod不需要更新或有其他编排等。</p>
<h4 id="1-1-2-1-ReplicaSet-示例"><a href="#1-1-2-1-ReplicaSet-示例" class="headerlink" title="1.1.2.1 ReplicaSet 示例"></a>1.1.2.1 ReplicaSet 示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ReplicaSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">frontend</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">guestbook</span></span><br><span class="line">    <span class="attr">tier:</span> <span class="string">frontend</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="comment"># modify replicas according to your case</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">3</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">tier:</span> <span class="string">frontend</span></span><br><span class="line">    <span class="attr">matchExpressions:</span></span><br><span class="line">      <span class="bullet">-</span> &#123;<span class="attr">key:</span> <span class="string">tier</span>, <span class="attr">operator:</span> <span class="string">In</span>, <span class="attr">values:</span> [<span class="string">frontend</span>]&#125;</span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">guestbook</span></span><br><span class="line">        <span class="attr">tier:</span> <span class="string">frontend</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">php-redis</span></span><br><span class="line">        <span class="attr">image:</span> <span class="string">gcr.io/google_samples/gb-frontend:v3</span></span><br><span class="line">        <span class="attr">resources:</span></span><br><span class="line">          <span class="attr">requests:</span></span><br><span class="line">            <span class="attr">cpu:</span> <span class="string">100m</span></span><br><span class="line">            <span class="attr">memory:</span> <span class="string">100Mi</span></span><br><span class="line">        <span class="attr">env:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">GET_HOSTS_FROM</span></span><br><span class="line">          <span class="attr">value:</span> <span class="string">dns</span></span><br><span class="line">          <span class="comment"># If your cluster config does not include a dns service, then to</span></span><br><span class="line">          <span class="comment"># instead access environment variables to find service host</span></span><br><span class="line">          <span class="comment"># info, comment out the &#x27;value: dns&#x27; line above, and uncomment the</span></span><br><span class="line">          <span class="comment"># line below.</span></span><br><span class="line">          <span class="comment"># value: env</span></span><br><span class="line">        <span class="attr">ports:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">containerPort:</span> <span class="number">80</span></span><br></pre></td></tr></table></figure>



<blockquote>
<p>Replication Controller和ReplicaSet的创建删除和Pod并无太大区别，Replication Controller目前几乎已经不在生产环境中使用，ReplicaSet也很少单独被使用，都是使用更高级的资源Deployment、DaemonSet、StatefulSet进行管理Pod</p>
</blockquote>
<h2 id="1-2-Deployment"><a href="#1-2-Deployment" class="headerlink" title="1.2 Deployment"></a>1.2 Deployment</h2><h3 id="1-2-1-Deployment概念"><a href="#1-2-1-Deployment概念" class="headerlink" title="1.2.1 Deployment概念"></a>1.2.1 Deployment概念</h3><p>用于部署无状态的服务，这个最常用的控制器。一般用于管理维护企业内部无状态的微服务，比如configserver、zuul、springboot。他可以管理多个副本的Pod实现无缝迁移、自动扩容缩容、自动灾难恢复、一键回滚等功能。</p>
<p>虽然ReplicaSet可以确保在任何给定时间运行的Pod副本达到指定的数量，但是Deployment（部署）是一个更高级的概念，它管理ReplicaSet并为Pod和ReplicaSet提供声明性更新以及许多其他有用的功能，所以建议在实际使用中，使用Deployment代替ReplicaSet。</p>
<p>如果在Deployment对象中描述了所需的状态，Deployment控制器就会以可控制的速率将实际状态更改为期望状态。也可以在Deployment中创建新的ReplicaSet，或者删除现有的Deployment并使用新的Deployment部署所用的资源。</p>
<h3 id="1-2-2-创建一个Deployment"><a href="#1-2-2-创建一个Deployment" class="headerlink" title="1.2.2  创建一个Deployment"></a>1.2.2  创建一个Deployment</h3><h4 id="1-2-2-1-命令行创建Deployment"><a href="#1-2-2-1-命令行创建Deployment" class="headerlink" title="1.2.2.1 命令行创建Deployment"></a>1.2.2.1 命令行创建Deployment</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create deployment nginx --image&#x3D;nginx:1.15.2</span><br></pre></td></tr></table></figure>



<h4 id="1-2-2-2-从文件创建Deployment"><a href="#1-2-2-2-从文件创建Deployment" class="headerlink" title="1.2.2.2 从文件创建Deployment"></a>1.2.2.2 从文件创建Deployment</h4><ul>
<li>创建nginx-deploy.yaml文件</li>
</ul>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Deployment</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">annotations:</span></span><br><span class="line">    <span class="attr">deployment.kubernetes.io/revision:</span> <span class="string">&quot;1&quot;</span></span><br><span class="line">  <span class="attr">generation:</span> <span class="number">1</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">progressDeadlineSeconds:</span> <span class="number">600</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">2</span> <span class="comment">#副本数</span></span><br><span class="line">  <span class="attr">revisionHistoryLimit:</span> <span class="number">10</span> <span class="comment"># 历史记录保留的个数</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">strategy:</span></span><br><span class="line">    <span class="attr">rollingUpdate:</span></span><br><span class="line">      <span class="attr">maxSurge:</span> <span class="number">25</span><span class="string">%</span></span><br><span class="line">      <span class="attr">maxUnavailable:</span> <span class="number">25</span><span class="string">%</span></span><br><span class="line">    <span class="attr">type:</span> <span class="string">RollingUpdate</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">creationTimestamp:</span> <span class="literal">null</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">resources:</span> &#123;&#125;</span><br><span class="line">        <span class="attr">terminationMessagePath:</span> <span class="string">/dev/termination-log</span></span><br><span class="line">        <span class="attr">terminationMessagePolicy:</span> <span class="string">File</span></span><br><span class="line">      <span class="attr">dnsPolicy:</span> <span class="string">ClusterFirst</span></span><br><span class="line">      <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line">      <span class="attr">schedulerName:</span> <span class="string">default-scheduler</span></span><br><span class="line">      <span class="attr">securityContext:</span> &#123;&#125;</span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br></pre></td></tr></table></figure>

<p>其中：</p>
<ol>
<li><p>nginx-deployment：Deployment的名称。</p>
</li>
<li><p>replicas： 创建Pod的副本数。</p>
</li>
<li><p>selector：定义Deployment如何找到要管理的Pod，与template的label（标签）对应。</p>
</li>
<li><p>template字段包含以下字段：</p>
<ul>
<li>app: nginx使用label（标签）标记Pod</li>
<li>spec：表示Pod运行一个名字为nginx的容器。</li>
<li>image：运行此Pod使用的镜像</li>
<li>Port：容器用于发送和接收流量的端口</li>
</ul>
</li>
</ol>
<ul>
<li><p>从文件创建deployment</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f nginx-deploy.yaml</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="1-2-2-3-deloyment状态解析"><a href="#1-2-2-3-deloyment状态解析" class="headerlink" title="1.2.2.3 deloyment状态解析"></a>1.2.2.3 deloyment状态解析</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get deploy -owide</span><br></pre></td></tr></table></figure>

<p>显示结果如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">NAME    READY   UP-TO-DATE   AVAILABLE   AGE    CONTAINERS   IMAGES         SELECTOR</span><br><span class="line">nginx   1&#x2F;1     1            1           6m4s   nginx        nginx:1.15.2   app&#x3D;nginx</span><br></pre></td></tr></table></figure>

<ul>
<li><p>NAME： Deployment名称</p>
</li>
<li><p>READY：Pod的状态，已经Ready的个数</p>
</li>
<li><p>UP-TO-DATE：已经达到期望状态的被更新的副本数</p>
</li>
<li><p>AVAILABLE：已经可以用的副本数</p>
</li>
<li><p>AGE：显示应用程序运行的时间</p>
</li>
<li><p>CONTAINERS：容器名称</p>
</li>
<li><p>IMAGES：容器的镜像</p>
</li>
<li><p>SELECTOR：管理的Pod的标签</p>
</li>
</ul>
<h3 id="1-2-3-Deployment的更新"><a href="#1-2-3-Deployment的更新" class="headerlink" title="1.2.3  Deployment的更新"></a>1.2.3  Deployment的更新</h3><p>一般对应用程序升级或者版本迭代时，会通过Deployment对Pod进行滚动更新</p>
<h4 id="1-2-3-1-更新Deployment示例"><a href="#1-2-3-1-更新Deployment示例" class="headerlink" title="1.2.3.1 更新Deployment示例"></a>1.2.3.1 更新Deployment示例</h4><h5 id="1-2-3-1-1-kubectl-set命令行更新"><a href="#1-2-3-1-1-kubectl-set命令行更新" class="headerlink" title="1.2.3.1.1 kubectl set命令行更新"></a>1.2.3.1.1 kubectl set命令行更新</h5><p>将nginx镜像从1.15.2，更新为1.15.3</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl set image deploy nginx nginx&#x3D;nginx:1.15.3 –-record</span><br></pre></td></tr></table></figure>

<ol>
<li>–record :  记录操作</li>
</ol>
<h5 id="1-2-3-1-2-查看更新状态"><a href="#1-2-3-1-2-查看更新状态" class="headerlink" title="1.2.3.1.2  查看更新状态"></a>1.2.3.1.2  查看更新状态</h5><ul>
<li><p>使用kubectl rollout status</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout status deploy nginx</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>使用describe查看</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl describe deploy nginx</span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="1-2-4-Deployment回滚"><a href="#1-2-4-Deployment回滚" class="headerlink" title="1.2.4 Deployment回滚"></a>1.2.4 Deployment回滚</h3><h4 id="1-2-4-1-查看历史更新记录"><a href="#1-2-4-1-查看历史更新记录" class="headerlink" title="1.2.4.1 查看历史更新记录"></a>1.2.4.1 查看历史更新记录</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout history deploy nginx</span><br></pre></td></tr></table></figure>



<h4 id="1-2-4-2-查看指定版本的详细信息"><a href="#1-2-4-2-查看指定版本的详细信息" class="headerlink" title="1.2.4.2 查看指定版本的详细信息"></a>1.2.4.2 查看指定版本的详细信息</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout history deploy nginx --revision&#x3D;5</span><br></pre></td></tr></table></figure>



<h4 id="1-2-4-3-回滚到上一个版本"><a href="#1-2-4-3-回滚到上一个版本" class="headerlink" title="1.2.4.3 回滚到上一个版本"></a>1.2.4.3 回滚到上一个版本</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout undo deploy nginx </span><br></pre></td></tr></table></figure>



<h4 id="1-2-4-4-回滚到指定版本"><a href="#1-2-4-4-回滚到指定版本" class="headerlink" title="1.2.4.4 回滚到指定版本"></a>1.2.4.4 回滚到指定版本</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout undo deploy nginx --to-revision&#x3D;5</span><br></pre></td></tr></table></figure>





<h3 id="1-2-5-Deployment-扩缩容"><a href="#1-2-5-Deployment-扩缩容" class="headerlink" title="1.2.5 Deployment 扩缩容"></a>1.2.5 Deployment 扩缩容</h3><h4 id="1-2-5-1-kubectl-scale命令实现扩缩容"><a href="#1-2-5-1-kubectl-scale命令实现扩缩容" class="headerlink" title="1.2.5.1 kubectl scale命令实现扩缩容"></a>1.2.5.1 kubectl scale命令实现扩缩容</h4><p>使用scale命令扩缩容，比直接edit编辑更方便</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl scale --replicas&#x3D;3 deploy nginx</span><br></pre></td></tr></table></figure>





<h3 id="1-2-6-Deployment的暂停和恢复"><a href="#1-2-6-Deployment的暂停和恢复" class="headerlink" title="1.2.6 Deployment的暂停和恢复"></a>1.2.6 Deployment的暂停和恢复</h3><p>如果有多条set命令需要更新，每使用set命令变更deployment时都会触发一次更新，这样会很不方便。使用Deployment 暂停功能可以暂停更新，等多个set命令执行完毕后，恢复更新。这样就只会触发一次包含之前所有set变更的更新。</p>
<h4 id="1-2-6-1-Deployment-暂停功能"><a href="#1-2-6-1-Deployment-暂停功能" class="headerlink" title="1.2.6.1 Deployment 暂停功能"></a>1.2.6.1 Deployment 暂停功能</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout pause deployment nginx</span><br></pre></td></tr></table></figure>

<h4 id="1-2-6-2-Deployment-恢复更新"><a href="#1-2-6-2-Deployment-恢复更新" class="headerlink" title="1.2.6.2 Deployment 恢复更新"></a>1.2.6.2 Deployment 恢复更新</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout resume deploy nginx </span><br></pre></td></tr></table></figure>



<h4 id="1-2-6-3-暂停和恢复示例"><a href="#1-2-6-3-暂停和恢复示例" class="headerlink" title="1.2.6.3 暂停和恢复示例"></a>1.2.6.3 暂停和恢复示例</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"># 暂停更新</span><br><span class="line">kubectl rollout pause deployment nginx</span><br><span class="line"></span><br><span class="line"># 第一次变更镜像</span><br><span class="line">kubectl set image deploy nginx nginx&#x3D;nginx:1.15.3 --record</span><br><span class="line"></span><br><span class="line"># 第二次变更cpu和内存，变更后查看yaml配置已经变更，但pod实际没有被更新</span><br><span class="line">kubectl set resources deploy nginx -c nginx --limits&#x3D;cpu&#x3D;200m,memory&#x3D;128Mi --requests&#x3D;cpu&#x3D;10m,memory&#x3D;16Mi</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># 恢复更新，更新生效</span><br><span class="line">kubectl rollout resume deploy nginx </span><br></pre></td></tr></table></figure>



<h3 id="1-2-7-Deployment更新注意事项"><a href="#1-2-7-Deployment更新注意事项" class="headerlink" title="1.2.7  Deployment更新注意事项"></a>1.2.7  Deployment更新注意事项</h3><ul>
<li><p>以下面的yaml文件为例：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line"># kubectl get deploy nginx -oyaml</span><br><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  annotations:</span><br><span class="line">    deployment.kubernetes.io&#x2F;revision: &quot;12&quot;</span><br><span class="line">    kubernetes.io&#x2F;change-cause: kubectl set image deploy nginx nginx&#x3D;nginx:1.15.3</span><br><span class="line">      --record&#x3D;true</span><br><span class="line">  generation: 19</span><br><span class="line">  labels:</span><br><span class="line">    app: nginx</span><br><span class="line">  name: nginx</span><br><span class="line">  namespace: default</span><br><span class="line">spec:</span><br><span class="line">  progressDeadlineSeconds: 600</span><br><span class="line">  replicas: 2</span><br><span class="line">  revisionHistoryLimit: 10</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      app: nginx</span><br><span class="line">  strategy:</span><br><span class="line">    rollingUpdate:</span><br><span class="line">      maxSurge: 25%</span><br><span class="line">      maxUnavailable: 25%</span><br><span class="line">    type: RollingUpdate</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      creationTimestamp: null</span><br><span class="line">      labels:</span><br><span class="line">        app: nginx</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">      - image: nginx:1.15.3</span><br><span class="line">        imagePullPolicy: IfNotPresent</span><br><span class="line">        name: nginx</span><br><span class="line">        resources:</span><br><span class="line">          limits:</span><br><span class="line">            cpu: 200m</span><br><span class="line">            memory: 128Mi</span><br><span class="line">          requests:</span><br><span class="line">            cpu: 10m</span><br><span class="line">            memory: 16Mi</span><br><span class="line">        terminationMessagePath: &#x2F;dev&#x2F;termination-log</span><br><span class="line">        terminationMessagePolicy: File</span><br><span class="line">      dnsPolicy: ClusterFirst</span><br><span class="line">      restartPolicy: Always</span><br><span class="line">      schedulerName: default-scheduler</span><br><span class="line">      securityContext: &#123;&#125;</span><br><span class="line">      terminationGracePeriodSeconds: 30</span><br></pre></td></tr></table></figure>

<ul>
<li><p>.spec.revisionHistoryLimit：设置保留RS旧的revision的个数，设置为0的话，不保留历史数据</p>
</li>
<li><p>.spec.minReadySeconds：可选参数，指定新创建的Pod在没有任何容器崩溃的情况下视为Ready最小的秒数，默认为0，即一旦被创建就视为可用。</p>
</li>
<li><p>滚动更新的策略：</p>
<ul>
<li><p>.spec.strategy.type：更新deployment的方式，默认是RollingUpdate</p>
<ol>
<li><p>Recreate:   表示重建，先删掉旧的Pod再创建新的Pod。</p>
</li>
<li><p>RollingUpdate：滚动更新，可以指定maxSurge和maxUnavailable</p>
</li>
</ol>
<blockquote>
<p>maxUnavailable：指定在回滚或更新时最大不可用的Pod的数量，可选字段，默认25%，可以设置成数字或百分比，如果该值为0，那么maxSurge就不能0</p>
</blockquote>
<blockquote>
<p>maxSurge：可以超过期望值的最大Pod数，可选字段，默认为25%，可以设置成数字或百分比，如果该值为0，那么maxUnavailable不能为0</p>
</blockquote>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<h2 id="1-2-8-总结"><a href="#1-2-8-总结" class="headerlink" title="1.2.8 总结"></a>1.2.8 总结</h2><p>Deployment 实际上是一个两层控制器。首先，它通过 ReplicaSet 的个数来描述应用的版本；然后，它再通过 ReplicaSet 的属性（比如 replicas 的值），来保证 Pod 的副本数量。</p>
<blockquote>
<p>备注：Deployment 控制 ReplicaSet（版本），ReplicaSet 控制 Pod（副本数）。这个两层控制关系一定要牢记。</p>
</blockquote>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">Deployment</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6RC&RS&Deployment/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s进阶权限控制" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E8%BF%9B%E9%98%B6%E6%9D%83%E9%99%90%E6%8E%A7%E5%88%B6/">K8s进阶权限控制</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E8%BF%9B%E9%98%B6%E6%9D%83%E9%99%90%E6%8E%A7%E5%88%B6/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="RBAC访问控制"><a href="#RBAC访问控制" class="headerlink" title="RBAC访问控制"></a>RBAC访问控制</h1><p>官方中文文档:<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding">https://kubernetes.io/zh/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding</a></p>
<h2 id="1-什么是RBAC"><a href="#1-什么是RBAC" class="headerlink" title="1 什么是RBAC"></a>1 什么是RBAC</h2><p>基于角色（Role）的访问控制（RBAC）是一种基于组织中用户的角色来调节控制对 计算机或网络资源的访问的方法。</p>
<p>Jenkins也可以使用基于角色的用户权限管理。</p>
<h2 id="2-RBAC中4个顶级资源"><a href="#2-RBAC中4个顶级资源" class="headerlink" title="2 RBAC中4个顶级资源"></a>2 RBAC中4个顶级资源</h2><p>Role：角色，包含一组权限的规则。没有拒绝规则，只是附加允许。Namespace隔离，只作用于命名空间内。</p>
<p>ClusterRole：和Role的区别，Role是只作用于命名空间内，作用于整个集群。</p>
<p>RoleBinding：作用于命令空间内，将ClusterRole或者Role绑定到User、Group、ServiceAccount。</p>
<p>ClusterRoleBinding：作用于整个集群。</p>
<h2 id="3-Role-和-ClusterRole"><a href="#3-Role-和-ClusterRole" class="headerlink" title="3 Role 和 ClusterRole"></a>3 Role 和 ClusterRole</h2><p>RBAC 的 <em>Role</em> 或 <em>ClusterRole</em> 中包含一组代表相关权限的规则。 这些权限是纯粹累加的（不存在拒绝某操作的规则）。</p>
<p>Role 总是用来在某个命名空间内设置访问权限；在你创建 Role 时，你必须指定该 Role 所属的名字空间。</p>
<p>与之相对，ClusterRole 则是一个集群作用域的资源。这两种资源的名字不同（Role 和 ClusterRole）是因为 Kubernetes 对象要么是名字空间作用域的，要么是集群作用域的， 不可两者兼具。</p>
<p>ClusterRole 有若干用法。你可以用它来：</p>
<ol>
<li>定义对某名字空间域对象的访问权限，并在各个命名空间内完成授权；</li>
<li>为命名空间作用域的对象设置访问权限，并跨所有命名空间执行授权；</li>
<li>为集群作用域的资源定义访问权限。</li>
</ol>
<p><strong>如果你希望在名字空间内定义角色，应该使用 Role； 如果你希望定义集群范围的角色，应该使用 ClusterRole</strong>。</p>
<h3 id="3-1-Role-示例"><a href="#3-1-Role-示例" class="headerlink" title="3.1 Role 示例"></a>3.1 Role 示例</h3><p>下面是一个位于 “default” 名字空间的 Role 的示例，可用来授予对 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/">pods</a> 的读访问权限：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Role</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">pod-reader</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;&quot;</span>] <span class="comment"># &quot;&quot; 标明 core API 组</span></span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;pods&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;watch&quot;</span>, <span class="string">&quot;list&quot;</span>]</span><br></pre></td></tr></table></figure>



<h3 id="3-2-ClusterRole-示例"><a href="#3-2-ClusterRole-示例" class="headerlink" title="3.2 ClusterRole 示例"></a>3.2 ClusterRole 示例</h3><p>ClusterRole 可以和 Role 相同完成授权。 因为 ClusterRole 属于集群范围，所以它也可以为以下资源授予访问权限：</p>
<ul>
<li>集群范围资源（比如 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/architecture/nodes/">节点（Node）</a>）</li>
<li>非资源端点（比如 <code>/healthz</code>）</li>
<li>跨命名空间访问的命名空间作用域的资源（如 Pods），比如，你可以使用 ClusterRole 来允许某特定用户执行 <code>kubectl get pods --all-namespaces</code></li>
</ul>
<p>下面是一个 ClusterRole 的示例，可用来为任一特定名字空间中的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/configuration/secret/">Secret</a> 授予读访问权限， 或者跨名字空间的访问权限（取决于该角色是如何<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding">绑定</a>的）：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="comment"># &quot;namespace&quot; 被忽略，因为 ClusterRoles 不受名字空间限制</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-reader</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;&quot;</span>]</span><br><span class="line">  <span class="comment"># 在 HTTP 层面，用来访问 Secret 对象的资源的名称为 &quot;secrets&quot;</span></span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;secrets&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;watch&quot;</span>, <span class="string">&quot;list&quot;</span>]</span><br></pre></td></tr></table></figure>



<h2 id="4-RoleBinding-和-ClusterRoleBindin"><a href="#4-RoleBinding-和-ClusterRoleBindin" class="headerlink" title="4 RoleBinding 和 ClusterRoleBindin"></a>4 RoleBinding 和 ClusterRoleBindin</h2><p>角色绑定（Role Binding）是将角色中定义的权限赋予一个或者一组用户。 它包含若干 <strong>主体</strong>（用户、组或服务账户）的列表和对这些主体所获得的角色的引用。 RoleBinding 在指定的名字空间中执行授权，而 ClusterRoleBinding 在集群范围执行授权。</p>
<p>一个 RoleBinding 可以引用同一的名字空间中的任何 Role。 或者，一个 RoleBinding 可以引用某 ClusterRole 并将该 ClusterRole 绑定到 RoleBinding 所在的名字空间。 如果你希望将某 ClusterRole 绑定到集群中所有名字空间，你要使用 ClusterRoleBinding。</p>
<h3 id="4-1-RoleBinding-示例"><a href="#4-1-RoleBinding-示例" class="headerlink" title="4.1 RoleBinding 示例"></a>4.1 RoleBinding 示例</h3><p>下面的例子中的 RoleBinding 将 “pod-reader” Role 授予在 “default” 名字空间中的用户 “jane”。 这样，用户 “jane” 就具有了读取 “default” 名字空间中 pods 的权限。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="comment"># 此角色绑定允许 &quot;jane&quot; 读取 &quot;default&quot; 名字空间中的 Pods</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">RoleBinding</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">read-pods</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="comment"># 你可以指定不止一个“subject（主体）”</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">User</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">jane</span> <span class="comment"># &quot;name&quot; 是不区分大小写的</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br><span class="line"><span class="attr">roleRef:</span></span><br><span class="line">  <span class="comment"># &quot;roleRef&quot; 指定与某 Role 或 ClusterRole 的绑定关系</span></span><br><span class="line">  <span class="attr">kind:</span> <span class="string">Role</span> <span class="comment"># 此字段必须是 Role 或 ClusterRole</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">pod-reader</span>     <span class="comment"># 此字段必须与你要绑定的 Role 或 ClusterRole 的名称匹配</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>



<p>RoleBinding 也可以引用 ClusterRole，以将对应 ClusterRole 中定义的访问权限授予 RoleBinding 所在名字空间的资源。这种引用使得你可以跨整个集群定义一组通用的角色， 之后在多个名字空间中复用。</p>
<p>例如，尽管下面的 RoleBinding 引用的是一个 ClusterRole，”dave”（这里的主体， 不区分大小写）只能访问 “development” 名字空间中的 Secrets 对象，因为 RoleBinding 所在的名字空间（由其 metadata 决定）是 “development”。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="comment"># 此角色绑定使得用户 &quot;dave&quot; 能够读取 &quot;default&quot; 名字空间中的 Secrets</span></span><br><span class="line"><span class="comment"># 你需要一个名为 &quot;secret-reader&quot; 的 ClusterRole</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">RoleBinding</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">read-secrets</span></span><br><span class="line">  <span class="comment"># RoleBinding 的名字空间决定了访问权限的授予范围。</span></span><br><span class="line">  <span class="comment"># 这里仅授权在 &quot;development&quot; 名字空间内的访问权限。</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">development</span></span><br><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">User</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dave</span> <span class="comment"># &#x27;name&#x27; 是不区分大小写的</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br><span class="line"><span class="attr">roleRef:</span></span><br><span class="line">  <span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-reader</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>



<h3 id="4-2-ClusterRoleBinding-示例"><a href="#4-2-ClusterRoleBinding-示例" class="headerlink" title="4.2 ClusterRoleBinding 示例"></a>4.2 ClusterRoleBinding 示例</h3><p>要跨整个集群完成访问权限的授予，你可以使用一个 ClusterRoleBinding。 下面的 ClusterRoleBinding 允许 “manager” 组内的所有用户访问任何名字空间中的 Secrets。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="comment"># 此集群角色绑定允许 “manager” 组中的任何人访问任何名字空间中的 secrets</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRoleBinding</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">read-secrets-global</span></span><br><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">manager</span> <span class="comment"># &#x27;name&#x27; 是不区分大小写的</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br><span class="line"><span class="attr">roleRef:</span></span><br><span class="line">  <span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-reader</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>



<h3 id="4-3-Role和ClusterRole更新限制"><a href="#4-3-Role和ClusterRole更新限制" class="headerlink" title="4.3 Role和ClusterRole更新限制"></a>4.3 Role和ClusterRole更新限制</h3><p>创建了绑定之后，你不能再修改绑定对象所引用的 Role 或 ClusterRole。 如果你想要改变现有绑定对象中 <code>roleRef</code> 字段的内容，必须删除重新创建绑定对象。</p>
<p>这种限制有两个主要原因：</p>
<ol>
<li>针对不同角色的绑定是完全不一样的绑定。要求通过删除/重建绑定来更改 <code>roleRef</code>, 这样可以确保要赋予绑定的所有主体会被授予新的角色（而不是在允许修改 <code>roleRef</code> 的情况下导致所有现有主体未经验证即被授予新角色对应的权限）。</li>
<li>将 <code>roleRef</code> 设置为不可以改变，这使得可以为用户授予对现有绑定对象的 <code>update</code> 权限， 这样可以让他们管理主体列表，同时不能更改被授予这些主体的角色。</li>
</ol>
<p>命令 <code>kubectl auth reconcile</code> 可以创建或者更新包含 RBAC 对象的清单文件， 并且在必要的情况下删除和重新创建绑定对象，以改变所引用的角色。 更多相关信息请参照<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/rbac/#kubectl-auth-reconcile">命令用法和示例</a></p>
<h2 id="5-对资源-resources-的引用"><a href="#5-对资源-resources-的引用" class="headerlink" title="5 对资源(resources)的引用"></a>5 对资源(resources)的引用</h2><p>在 Kubernetes API 中，大多数资源都是使用对象名称的字符串表示来呈现与访问的。 例如，对于 Pod 应使用 “pods”。 RBAC 使用对应 API 端点的 URL 中呈现的名字来引用资源。 有一些 Kubernetes API 涉及 <strong>子资源（subresource）</strong>，例如 Pod 的日志。 对 Pod 日志的请求看起来像这样：</p>
<figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">GET /api/v1/namespaces/&#123;namespace&#125;/pods/&#123;name&#125;/log</span><br></pre></td></tr></table></figure>

<p>在这里，<code>pods</code> 对应名字空间作用域的 Pod 资源，而 <code>log</code> 是 <code>pods</code> 的子资源。 在 RBAC 角色表达子资源时，使用斜线（<code>/</code>）来分隔资源和子资源。 要允许某主体读取 <code>pods</code> 同时访问这些 Pod 的 <code>log</code> 子资源，你可以这么写：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Role</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">pod-and-pod-logs-reader</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;&quot;</span>]</span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;pods&quot;</span>, <span class="string">&quot;pods/log&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;list&quot;</span>]</span><br></pre></td></tr></table></figure>

<p>对于某些请求，也可以通过 <code>resourceNames</code> 列表按名称引用资源。 在指定时，可以将请求限定为资源的单个实例。 下面的例子中限制可以 “get” 和 “update” 一个名为 <code>my-configmap</code> 的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-pod-configmap/">ConfigMap</a>：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Role</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">configmap-updater</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;&quot;</span>]</span><br><span class="line">  <span class="comment"># 在 HTTP 层面，用来访问 ConfigMap 的资源的名称为 &quot;configmaps&quot;</span></span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;configmaps&quot;</span>]</span><br><span class="line">  <span class="attr">resourceNames:</span> [<span class="string">&quot;my-configmap&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;update&quot;</span>, <span class="string">&quot;get&quot;</span>]</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>说明：</strong></p>
<p>你不能针对 <code>create</code> 或者 <code>deletecollection</code> 请求来实施 resourceName 限制。 对于 <code>create</code> 操作而言，这是因为在鉴权时还不知道对象名称。</p>
</blockquote>
<h2 id="6-聚合的-ClusterRole"><a href="#6-聚合的-ClusterRole" class="headerlink" title="6 聚合的 ClusterRole"></a>6 聚合的 ClusterRole</h2><p>你可以将若干 ClusterRole <strong>聚合（Aggregate）</strong> 起来，形成一个复合的 ClusterRole。 某个控制器作为集群控制面的一部分会监视带有 <code>aggregationRule</code> 的 ClusterRole 对象集合。<code>aggregationRule</code> 为控制器定义一个标签 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/labels/">选择算符</a>供后者匹配 应该组合到当前 ClusterRole 的 <code>roles</code> 字段中的 ClusterRole 对象。</p>
<p>下面是一个聚合 ClusterRole 的示例：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">monitoring</span></span><br><span class="line"><span class="attr">aggregationRule:</span></span><br><span class="line">  <span class="attr">clusterRoleSelectors:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">rbac.example.com/aggregate-to-monitoring:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line"><span class="attr">rules:</span> [] <span class="comment"># 控制面自动填充这里的规则</span></span><br></pre></td></tr></table></figure>

<p>当创建一个与现有聚合ClusterRole的标签选择算符匹配的ClusterRole，就会触发新的规则被添加到聚合 ClusterRole 的操作。</p>
<p>下面的例子中，通过创建一个标签同样为 <code>rbac.example.com/aggregate-to-monitoring: true</code> 的 ClusterRole，新的规则可被添加到 “monitoring” ClusterRole 中。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">monitoring-endpoints</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">rbac.example.com/aggregate-to-monitoring:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line"><span class="comment"># 当你创建 &quot;monitoring-endpoints&quot; ClusterRole 时，</span></span><br><span class="line"><span class="comment"># 下面的规则会被添加到 &quot;monitoring&quot; ClusterRole 中</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;&quot;</span>]</span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;services&quot;</span>, <span class="string">&quot;endpoints&quot;</span>, <span class="string">&quot;pods&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;list&quot;</span>, <span class="string">&quot;watch&quot;</span>]</span><br></pre></td></tr></table></figure>

<p>默认的<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings">面向用户的角色</a> 使用 ClusterRole 聚合。 这使得作为集群管理员的你可以为扩展默认规则，包括为定制资源设置规则， 比如通过 CustomResourceDefinitions 或聚合 API 服务器提供的定制资源。</p>
<p>例如，下面的 ClusterRoles 让默认角色 “admin” 和 “edit” 拥有管理自定义资源 “CronTabs” 的权限， “view” 角色对 CronTab 资源拥有读操作权限。 你可以假定 CronTab 对象在 API 服务器所看到的 URL 中被命名为 <code>&quot;crontabs&quot;</code>。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">aggregate-cron-tabs-edit</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="comment"># 添加以下权限到默认角色 &quot;admin&quot; 和 &quot;edit&quot; 中</span></span><br><span class="line">    <span class="attr">rbac.authorization.k8s.io/aggregate-to-admin:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line">    <span class="attr">rbac.authorization.k8s.io/aggregate-to-edit:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;stable.example.com&quot;</span>]</span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;crontabs&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;list&quot;</span>, <span class="string">&quot;watch&quot;</span>, <span class="string">&quot;create&quot;</span>, <span class="string">&quot;update&quot;</span>, <span class="string">&quot;patch&quot;</span>, <span class="string">&quot;delete&quot;</span>]</span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ClusterRole</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">rbac.authorization.k8s.io/v1</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">aggregate-cron-tabs-view</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="comment"># 添加以下权限到 &quot;view&quot; 默认角色中</span></span><br><span class="line">    <span class="attr">rbac.authorization.k8s.io/aggregate-to-view:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line"><span class="attr">rules:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">apiGroups:</span> [<span class="string">&quot;stable.example.com&quot;</span>]</span><br><span class="line">  <span class="attr">resources:</span> [<span class="string">&quot;crontabs&quot;</span>]</span><br><span class="line">  <span class="attr">verbs:</span> [<span class="string">&quot;get&quot;</span>, <span class="string">&quot;list&quot;</span>, <span class="string">&quot;watch&quot;</span>]</span><br></pre></td></tr></table></figure>





<h2 id="7-对主体Subject的引用"><a href="#7-对主体Subject的引用" class="headerlink" title="7 对主体Subject的引用"></a>7 对主体Subject的引用</h2><p>RoleBinding 或者 ClusterRoleBinding 可绑定角色到某 主体（Subject）上。 主体可以是组，用户或者 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-service-account/">服务账户</a>。</p>
<p>Kubernetes 用字符串来表示用户名。 用户名可以是普通的用户名，像 “alice”；或者是邮件风格的名称，如 “<a href="mailto:&#x62;&#111;&#x62;&#64;&#101;&#120;&#97;&#x6d;&#112;&#108;&#x65;&#x2e;&#x63;&#x6f;&#x6d;">&#x62;&#111;&#x62;&#64;&#101;&#120;&#97;&#x6d;&#112;&#108;&#x65;&#x2e;&#x63;&#x6f;&#x6d;</a>“， 或者是以字符串形式表达的数字 ID。 你作为 Kubernetes 管理员负责配置 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/authentication/">身份认证模块</a> 以便后者能够生成你所期望的格式的用户名。</p>
<blockquote>
<p><strong>注意：</strong></p>
<p>前缀 <code>system:</code> 是 Kubernetes 系统保留的，所以你要确保 所配置的用户名或者组名不能出现上述 <code>system:</code> 前缀。 除了对前缀的限制之外，RBAC 鉴权系统不对用户名格式作任何要求。</p>
</blockquote>
<p>在 Kubernetes 中，鉴权模块提供用户组信息。 与用户名一样，用户组名也用字符串来表示，而且对该字符串没有格式要求， 只是不能使用保留的前缀 <code>system:</code>。</p>
<p><a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-service-account/">服务账户</a> 的用户名前缀为 <code>system:serviceaccount:</code>，属于前缀为 <code>system:serviceaccounts:</code> 的用户组。</p>
<blockquote>
<p><strong>说明：</strong></p>
<ul>
<li><code>system:serviceaccount:</code> （单数）是用于服务账户用户名的前缀；</li>
<li><code>system:serviceaccounts:</code> （复数）是用于服务账户组名的前缀。</li>
</ul>
</blockquote>
<h4 id="7-1-RoleBinding-示例"><a href="#7-1-RoleBinding-示例" class="headerlink" title="7.1 RoleBinding 示例"></a>7.1 RoleBinding 示例</h4><p>下面示例是 <code>RoleBinding</code> 中的片段，仅展示其 <code>subjects</code> 的部分。</p>
<p>对于名称为 <code>alice@example.com</code> 的用户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">User</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">&quot;alice@example.com&quot;</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于名称为 <code>frontend-admins</code> 的用户组：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">&quot;frontend-admins&quot;</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于 <code>kube-system</code> 名字空间中的默认服务账户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">ServiceAccount</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">kube-system</span></span><br></pre></td></tr></table></figure>

<p>对于任何名称空间中的 “qa” 组中所有的服务账户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:serviceaccounts:qa</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于 “development” 名称空间中 “dev” 组中的所有服务帐户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:serviceaccounts:dev</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">development</span></span><br></pre></td></tr></table></figure>

<p>对于在任何名字空间中的服务账户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:serviceaccounts</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于所有已经过认证的用户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:authenticated</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于所有未通过认证的用户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:unauthenticated</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>

<p>对于所有用户：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">subjects:</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:authenticated</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br><span class="line"><span class="bullet">-</span> <span class="attr">kind:</span> <span class="string">Group</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">system:unauthenticated</span></span><br><span class="line">  <span class="attr">apiGroup:</span> <span class="string">rbac.authorization.k8s.io</span></span><br></pre></td></tr></table></figure>



      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">RBAC</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E8%BF%9B%E9%98%B6%E6%9D%83%E9%99%90%E6%8E%A7%E5%88%B6/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/ubuntu 18.04 安装kubeadm 1.20.4" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/ubuntu%2018.04%20%E5%AE%89%E8%A3%85kubeadm%201.20.4/">Ubuntu18.04 kubeadm安装k8s1.20.4(开发测试级)</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/ubuntu%2018.04%20%E5%AE%89%E8%A3%85kubeadm%201.20.4/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="ubuntu-18-04-安装kubeadm-1-20-4"><a href="#ubuntu-18-04-安装kubeadm-1-20-4" class="headerlink" title="ubuntu 18.04 安装kubeadm 1.20.4"></a>ubuntu 18.04 安装kubeadm 1.20.4</h1><h2 id="1-环境详情"><a href="#1-环境详情" class="headerlink" title="1 环境详情"></a>1 环境详情</h2><table>
<thead>
<tr>
<th>系统</th>
<th>内网IP</th>
<th>节点名称</th>
<th>配置</th>
<th>软件版本</th>
</tr>
</thead>
<tbody><tr>
<td>ubuntu18.04</td>
<td>192.168.31.100</td>
<td>master</td>
<td>2c4g</td>
<td>k8s：1.20.4   docker：19.03.11</td>
</tr>
<tr>
<td>ubuntu18.04</td>
<td>192.168.31.101</td>
<td>node01</td>
<td>2c4g</td>
<td>k8s：1.20.4   docker：19.03.11</td>
</tr>
<tr>
<td>ubuntu18.04</td>
<td>192.168.31.102</td>
<td>node02</td>
<td>2c4g</td>
<td>k8s：1.20.4   docker：19.03.11</td>
</tr>
</tbody></table>
<h2 id="2-安装docker"><a href="#2-安装docker" class="headerlink" title="2 安装docker"></a>2 安装docker</h2><p>官网：<a target="_blank" rel="noopener" href="https://v1-19.docs.kubernetes.io/zh/docs/setup/production-environment/container-runtimes/">https://v1-19.docs.kubernetes.io/zh/docs/setup/production-environment/container-runtimes/</a></p>
<p>以下操作需要在所有节点执行:</p>
<ul>
<li>安装依赖</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get update &amp;&amp; sudo apt-get install -y \</span><br><span class="line">  apt-transport-https ca-certificates curl software-properties-common gnupg2</span><br></pre></td></tr></table></figure>



<ul>
<li>新增 Docker 的 官方 GPG 秘钥:</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -fsSL https:&#x2F;&#x2F;download.docker.com&#x2F;linux&#x2F;ubuntu&#x2F;gpg | sudo apt-key add --keyring &#x2F;etc&#x2F;apt&#x2F;trusted.gpg.d&#x2F;docker.gpg -</span><br></pre></td></tr></table></figure>



<ul>
<li>添加 Docker apt 仓库:</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo add-apt-repository \</span><br><span class="line">  &quot;deb [arch&#x3D;amd64] https:&#x2F;&#x2F;download.docker.com&#x2F;linux&#x2F;ubuntu \</span><br><span class="line">  $(lsb_release -cs) \</span><br><span class="line">  stable&quot;</span><br></pre></td></tr></table></figure>



<ul>
<li>安装 Docker CE</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get update &amp;&amp; sudo apt-get install -y \</span><br><span class="line">  containerd.io&#x3D;1.2.13-2 \</span><br><span class="line">  docker-ce&#x3D;5:19.03.11~3-0~ubuntu-$(lsb_release -cs) \</span><br><span class="line">  docker-ce-cli&#x3D;5:19.03.11~3-0~ubuntu-$(lsb_release -cs)</span><br></pre></td></tr></table></figure>



<ul>
<li>设置 Docker daemon</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF | sudo tee &#x2F;etc&#x2F;docker&#x2F;daemon.json</span><br><span class="line">&#123;</span><br><span class="line">  &quot;exec-opts&quot;: [&quot;native.cgroupdriver&#x3D;systemd&quot;],</span><br><span class="line">  &quot;log-driver&quot;: &quot;json-file&quot;,</span><br><span class="line">  &quot;log-opts&quot;: &#123;</span><br><span class="line">    &quot;max-size&quot;: &quot;100m&quot;</span><br><span class="line">  &#125;,</span><br><span class="line">  &quot;storage-driver&quot;: &quot;overlay2&quot;</span><br><span class="line">&#125;</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



<ul>
<li>创建docker system</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sudo mkdir -p &#x2F;etc&#x2F;systemd&#x2F;system&#x2F;docker.service.d</span><br></pre></td></tr></table></figure>



<ul>
<li>重启 docker并设置开机启动</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">sudo systemctl daemon-reload</span><br><span class="line">sudo systemctl restart docker</span><br><span class="line">sudo systemctl enable --now docker</span><br></pre></td></tr></table></figure>





<h2 id="3-安装k8s组件"><a href="#3-安装k8s组件" class="headerlink" title="3 安装k8s组件"></a>3 安装k8s组件</h2><p>官网：<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/install-kubeadm/">https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/install-kubeadm/</a></p>
<ul>
<li>配置系统内核</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF | sudo tee &#x2F;etc&#x2F;modules-load.d&#x2F;k8s.conf</span><br><span class="line">br_netfilter</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line">cat &lt;&lt;EOF | sudo tee &#x2F;etc&#x2F;sysctl.d&#x2F;k8s.conf</span><br><span class="line">net.bridge.bridge-nf-call-ip6tables &#x3D; 1</span><br><span class="line">net.bridge.bridge-nf-call-iptables &#x3D; 1</span><br><span class="line">EOF</span><br><span class="line">sudo sysctl --system</span><br></pre></td></tr></table></figure>





<ul>
<li>更新 <code>apt</code> 包索引并安装使用 Kubernetes <code>apt</code> 仓库所需要的包：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get update</span><br><span class="line">sudo apt-get install -y apt-transport-https ca-certificates curl</span><br></pre></td></tr></table></figure>



<ul>
<li>添加国内源（官方源太慢)</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">curl https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;kubernetes&#x2F;apt&#x2F;doc&#x2F;apt-key.gpg | sudo apt-key add -</span><br><span class="line"></span><br><span class="line">cat &gt; &#x2F;etc&#x2F;apt&#x2F;sources.list.d&#x2F;kubernetes.list &lt;&lt; EOF </span><br><span class="line">deb https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;kubernetes&#x2F;apt kubernetes-xenial main</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



<ul>
<li><p>更新 <code>apt</code> 包索引，安装 kubelet、kubeadm 和 kubectl，并锁定其版本</p>
<blockquote>
<p>此处指定安装版本1.20.4</p>
</blockquote>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get update</span><br><span class="line">sudo apt-get install -y kubelet&#x3D;1.20.4-00 kubeadm&#x3D;1.20.4-00 kubectl&#x3D;1.20.4-00</span><br><span class="line">sudo apt-mark hold kubelet kubeadm kubectl</span><br></pre></td></tr></table></figure>





<h2 id="4-kubeadm初始化集群"><a href="#4-kubeadm初始化集群" class="headerlink" title="4 kubeadm初始化集群"></a>4 kubeadm初始化集群</h2><ul>
<li><p>使用国内源拉取基础镜像</p>
<blockquote>
<p>此处指定安装版本1.20.4</p>
<p>此操作在所有节点执行</p>
</blockquote>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubeadm config images pull \</span><br><span class="line">--image-repository registry.cn-hangzhou.aliyuncs.com&#x2F;google_containers \</span><br><span class="line">--kubernetes-version 1.20.4</span><br></pre></td></tr></table></figure>



<ul>
<li><p>初始化集群</p>
<blockquote>
<p>此处指定安装版本1.20.4</p>
<p>此操作仅在master节点执行</p>
</blockquote>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubeadm init --apiserver-advertise-address 192.168.31.100 \</span><br><span class="line">--image-repository registry.cn-hangzhou.aliyuncs.com&#x2F;google_containers \</span><br><span class="line">--kubernetes-version 1.20.4</span><br></pre></td></tr></table></figure>



<ul>
<li>执行完成后根据提示操作</li>
</ul>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424152126057.png" alt="image-20210424152126057"></p>
<p>在master节点，创建.kube/config：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">mkdir -p $HOME&#x2F;.kube</span><br><span class="line">sudo cp -i &#x2F;etc&#x2F;kubernetes&#x2F;admin.conf $HOME&#x2F;.kube&#x2F;config</span><br><span class="line">sudo chown $(id -u):$(id -g) $HOME&#x2F;.kube&#x2F;config</span><br></pre></td></tr></table></figure>



<p>在node节点，加入k8s集群：</p>
<blockquote>
<p>注意需要根据实际提示的结果操作</p>
</blockquote>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">kubeadm join 192.168.31.100:6443 --token 85iirq.if85o54yfvcrs7cv \</span><br><span class="line">    --discovery-token-ca-cert-hash sha256:db943aa4445ca87233b9025f9ad405d782b4601a5d922fee080717c9b42bcf9d</span><br></pre></td></tr></table></figure>

<p>如果token超过24小时过期后，需要重建可以使用下面的命令生成:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm token create --print-join-command</span><br></pre></td></tr></table></figure>



<h2 id="5-验证集群状态"><a href="#5-验证集群状态" class="headerlink" title="5 验证集群状态"></a>5 验证集群状态</h2><ul>
<li>在master节点执行命令：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get pod -n kube-system</span><br></pre></td></tr></table></figure>

<p>输出如下，即为正常状态：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">NAME                                 READY   STATUS    RESTARTS   AGE</span><br><span class="line">coredns-54d67798b7-lkchw             0&#x2F;1     Pending   0          5m24s</span><br><span class="line">coredns-54d67798b7-t4xpv             0&#x2F;1     Pending   0          5m24s</span><br><span class="line">etcd-cka-master                      1&#x2F;1     Running   0          5m31s</span><br><span class="line">kube-apiserver-cka-master            1&#x2F;1     Running   0          5m31s</span><br><span class="line">kube-controller-manager-cka-master   1&#x2F;1     Running   0          5m31s</span><br><span class="line">kube-proxy-5qd7t                     1&#x2F;1     Running   0          5m24s</span><br><span class="line">kube-proxy-pdmhq                     1&#x2F;1     Running   0          5m2s</span><br><span class="line">kube-proxy-wpzcj                     1&#x2F;1     Running   0          4m57s</span><br><span class="line">kube-scheduler-cka-master            1&#x2F;1     Running   0          5m31s</span><br></pre></td></tr></table></figure>

<blockquote>
<p>其中coredns为Pending，是因为还没有安装网络插件的缘故</p>
</blockquote>
<h2 id="6-安装插件"><a href="#6-安装插件" class="headerlink" title="6 安装插件"></a>6 安装插件</h2><p>插件的yaml下载地址：</p>
<blockquote>
<p>链接：<a target="_blank" rel="noopener" href="https://pan.xunlei.com/s/VMZ1rH-oMzmvI5q4WtFLxxvkA1">https://pan.xunlei.com/s/VMZ1rH-oMzmvI5q4WtFLxxvkA1</a><br>提取码：tdef</p>
</blockquote>
<ul>
<li><p>安装calico网络插件</p>
<blockquote>
<p>注意cluster-cidr的网段不能与宿主机和service网段重复</p>
</blockquote>
</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f calico.yaml</span><br></pre></td></tr></table></figure>





<ul>
<li>安装metrics</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f comp.yaml</span><br></pre></td></tr></table></figure>

<blockquote>
<ul>
<li>此时可能会出现pod创建失败的情况，查看pod状态如下</li>
</ul>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424160221910.png" alt="image-20210424160221910"></p>
<ul>
<li><p>检查pod日志，输入命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl logs -f metrics-server-545b8b99c6-ngq28  -n kube-system</span><br></pre></td></tr></table></figure>
</li>
<li><p>日志最上方发现报错：</p>
<p><code>Error: unable to create request header authentication config: open /etc/kubernetes/pki/front-proxy-ca.crt: no such file or directory</code></p>
<blockquote>
<p>根据日志可知是因为metrics所在节点没有<code>front-proxy-ca.crt</code>证书文件</p>
</blockquote>
</li>
<li><p>将master节点的<code>front-proxy-ca.crt</code>证书复制到所有节点：</p>
<blockquote>
<p>直接使用下面的命令可能会失败，因为Ubuntu不能使用密码访问root用户，可以设置免密或者使用普通用户拷贝到/tmp目录下后在复制过去</p>
</blockquote>
</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">for i in 192.168.31.101 192.168.31.102;do scp /etc/kubernetes/pki/front-proxy-ca.crt $i:/etc/kubernetes/pki/;done</span><br></pre></td></tr></table></figure>

<ul>
<li>删除所有资源后，重新创建</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete -f comp.yaml &amp;&amp; kubectl create -f comp.yaml</span><br></pre></td></tr></table></figure>
</blockquote>
<ul>
<li>安装dashboard</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f user.yaml</span><br><span class="line">kubectl create -f dashboard.yaml</span><br></pre></td></tr></table></figure>





<h2 id="7-访问dashboard"><a href="#7-访问dashboard" class="headerlink" title="7 访问dashboard"></a>7 访问dashboard</h2><ul>
<li>查看dashboard 的 service访问端口，用于在游览器中访问</li>
</ul>
<blockquote>
<p>此处端口为31592</p>
</blockquote>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424161404257.png" alt="image-20210424161404257"></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get svc -n kubernetes-dashboard</span><br></pre></td></tr></table></figure>



<ul>
<li><p>查看访问的token,用于登陆dashboard</p>
<ol>
<li><p>首先确认secret名称</p>
<blockquote>
<p>此处secret名称为：kubernetes-dashboard-token-lbnjg</p>
</blockquote>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424161608122.png" alt="image-20210424161608122"></p>
</li>
</ol>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get secrte -n kubernetes-dashboard</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ol start="2">
<li><p>查看token</p>
<blockquote>
<p>此处token为：eyJhbGciOiJSUzI1NiIsImtpZCI6IlE1RmtaWThXdjYwYWpSdlRxTEJMcEM2c3YweVlSV1B1SGhNMG1OZk90M1kifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi1sYm5qZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImUyMTY3MGZkLWQ0ZGMtNGU2Zi1iZWE1LWU5YTdhMTQ0OGMyNCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.OmGMA3JZ5sLeDeZgoMVD0zKWkX-oJG4tt6nZyC6w7-u6l_jsaW0LYr_dSdOnPTx47kNjio5G2ebFeDbCqbtQPGtprmd4NznUJbGKb0NIIQZ9V1w0wEkU81Je43EpyDBC2zA0iCOTEEgcoLQb0e3GJNKTus_a1lhkaH-09uY_-lQjHVPRNeF-nGzMkTmzinGhomFvHcmMwanMYXWyN_-k5hx9oxKTjDRkJgvk2izjZ9X8RHNWsmP__ztXzQVXZl8_fC6LtCCigIAW-ecYmpXSCFPoHiP8x3hdYjAm95P_YAL8x9p6HodcQkOHuOZ2QK7NDyM-JrmOLKZJ4h1l34qdsQ</p>
</blockquote>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424161808205.png" alt="image-20210424161808205"></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl describe secret kubernetes-dashboard-token-lbnjg -n kubernetes-dashboard</span><br></pre></td></tr></table></figure>



</li>
</ol>
<ol start="3">
<li><p>游览器访问</p>
<blockquote>
<p>必须使用https访问</p>
<p>地址可以是任意一个k8s节点的IP</p>
</blockquote>
</li>
</ol>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20210424162112871.png" alt="image-20210424162112871"></p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">k8s安装</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/ubuntu%2018.04%20%E5%AE%89%E8%A3%85kubeadm%201.20.4/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-日常笔记/gitee+hexo部署个人博客" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/%E6%97%A5%E5%B8%B8%E7%AC%94%E8%AE%B0/gitee+hexo%E9%83%A8%E7%BD%B2%E4%B8%AA%E4%BA%BA%E5%8D%9A%E5%AE%A2/">gitee+hexo部署个人博客</a>
    </h1>
  

        
        <a href="/2021/04/28/%E6%97%A5%E5%B8%B8%E7%AC%94%E8%AE%B0/gitee+hexo%E9%83%A8%E7%BD%B2%E4%B8%AA%E4%BA%BA%E5%8D%9A%E5%AE%A2/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2><ol>
<li><p>安装node.js</p>
</li>
<li><p>安装git</p>
</li>
<li><p>新建一个仓库</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/Linux%E6%80%A7%E8%83%BD%E4%BC%98%E5%8C%96%E5%AE%9E%E6%88%98/image-20200918153902728.png" alt="image-20200918153902728"></p>
<p><strong>注意：仓库路径填写用户名，后续pags域名则不会出现二级目录</strong></p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/Linux%E6%80%A7%E8%83%BD%E4%BC%98%E5%8C%96%E5%AE%9E%E6%88%98/image-20200918154022933.png" alt="image-20200918154022933"></p>
</li>
</ol>
        
      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">hexo</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/日常笔记//" class="article-tag-list-link color5">日常笔记</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/%E6%97%A5%E5%B8%B8%E7%AC%94%E8%AE%B0/gitee+hexo%E9%83%A8%E7%BD%B2%E4%B8%AA%E4%BA%BA%E5%8D%9A%E5%AE%A2/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-etcd/etcd 安装部署：单机和集群" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%20%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2%EF%BC%9A%E5%8D%95%E6%9C%BA%E5%92%8C%E9%9B%86%E7%BE%A4/">etcd安装部署:单机和集群</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%20%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2%EF%BC%9A%E5%8D%95%E6%9C%BA%E5%92%8C%E9%9B%86%E7%BE%A4/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-etcd-单机安装部署"><a href="#1-etcd-单机安装部署" class="headerlink" title="1 etcd 单机安装部署"></a>1 etcd 单机安装部署</h1><p>etcd 的安装有多种方式，这里我以 CentOS 7 为例，可以通过<code>yum install etcd</code>进行安装。然而通过系统工具安装的 etcd 版本比较滞后，如果需要安装最新版本的 etcd ，我们可以通过二进制包、源码编译以及 Docker 容器安装。</p>
<h2 id="1-1-二进制安装"><a href="#1-1-二进制安装" class="headerlink" title="1.1 二进制安装"></a>1.1 二进制安装</h2><p>目前最新的 etcd API 版本为 v3.4，我们基于 3.4.4 版本进行实践，API 版本与最新版保持一致，在 CentOS 7 上面</p>
<ul>
<li>使用如下脚本进行安装：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">ETCD_VER=v3.4.4</span><br><span class="line"></span><br><span class="line">GITHUB_URL=https://github.com/etcd-io/etcd/releases/download</span><br><span class="line"></span><br><span class="line">DOWNLOAD_URL=$&#123;GITHUB_URL&#125;</span><br><span class="line"></span><br><span class="line">rm -f /tmp/etcd-$&#123;ETCD_VER&#125;-linux-amd64.tar.gz</span><br><span class="line"></span><br><span class="line">rm -rf /tmp/etcd-download-test &amp;&amp; mkdir -p /tmp/etcd-download-test</span><br><span class="line"></span><br><span class="line">curl -L $&#123;DOWNLOAD_URL&#125;/$&#123;ETCD_VER&#125;/etcd-$&#123;ETCD_VER&#125;-linux-amd64.tar.gz -o /tmp/etcd-$&#123;ETCD_VER&#125;-linux-amd64.tar.gz</span><br><span class="line"></span><br><span class="line">tar xzvf /tmp/etcd-$&#123;ETCD_VER&#125;-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1</span><br><span class="line"></span><br><span class="line">rm -f /tmp/etcd-$&#123;ETCD_VER&#125;-linux-amd64.tar.gz</span><br><span class="line"></span><br><span class="line">/tmp/etcd-download-test/etcd --version</span><br><span class="line"></span><br><span class="line">/tmp/etcd-download-test/etcdctl version</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>下载可能比较慢，执行后，查看 etcd 版本的结果如下：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">etcd Version: 3.4.4</span><br><span class="line"></span><br><span class="line">Git SHA: e784ba73c</span><br><span class="line"></span><br><span class="line">Go Version: go1.12.12</span><br><span class="line"></span><br><span class="line">Go OS&#x2F;Arch: linux&#x2F;amd64</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>根据上面的执行结果可以看到，我们在 Linux 上安装成功，macOS 的二进制安装也类似，这里不重复演示了。关于 Windows 系统的安装比较简单，下载好安装包后，直接执行。其中 etcd.exe 是服务端，etcdctl.exe 是客户端</p>
<h2 id="1-2-源码安装"><a href="#1-2-源码安装" class="headerlink" title="1.2 源码安装"></a>1.2 源码安装</h2><p>使用源码安装，首先你需要确保本地的 Go 语言环境。如未安装，请参考 <a target="_blank" rel="noopener" href="https://golang.org/doc/install">https://golang.org/doc/install</a> 安装 Go 语言环境。我们需要 Go 版本为 1.13+，来构建最新版本的 etcd。</p>
<ul>
<li>首先查看一下我们本地的 Go 版本：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">$</span><span class="bash"> go version</span></span><br><span class="line"></span><br><span class="line">Go version go1.13.6 linux/amd64</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>经检测，本地的 Go 版本满足要求。将制定版本的 etcd 项目 clone 到本地之后，在项目文件夹构建 etcd。</p>
<ul>
<li>构建好之后，执行测试命令，确保 etcd 编译安装成功：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">$</span><span class="bash"> ./etcdctl version</span></span><br><span class="line"></span><br><span class="line">etcdctl version: 3.4.4</span><br><span class="line"></span><br><span class="line">API version: 3.4</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>经过上述步骤，我们已经通过源码编译成功安装 etcd。</p>
<p>除此之外，还可以使用 Docker 容器安装，这种方式更为简单，但这种方式有一个弊端：启动时会将 etcd 的端口暴露出来。</p>
<h1 id="2-etcd-集群安装部署"><a href="#2-etcd-集群安装部署" class="headerlink" title="2 etcd 集群安装部署"></a>2 etcd 集群安装部署</h1><p>刚刚我们介绍了 etcd 的单机安装，但是<strong>在实际生产环境中，为了整个集群的高可用，etcd 通常都会以集群方式部署，以避免单点故障</strong>。下面我们看看如何进行 etcd 集群部署。</p>
<p>引导 etcd 集群的启动有以下三种方式：</p>
<ul>
<li><p>静态启动</p>
</li>
<li><p>etcd 动态发现</p>
</li>
<li><p>DNS 发现</p>
</li>
</ul>
<p>其中静态启动 etcd 集群的方式要求每个成员都知道集群中的其他成员。然而很多场景中，群集成员的 IP 可能未知。因此需要借助发现服务引导 etcd 群集启动。下面我们将会分别介绍这几种方式。</p>
<h2 id="2-1-静态方式启动-etcd-集群"><a href="#2-1-静态方式启动-etcd-集群" class="headerlink" title="2.1 静态方式启动 etcd 集群"></a>2.1 静态方式启动 etcd 集群</h2><p>如果我们想在一台机器上实践 etcd 集群的搭建，可以通过 goreman 工具。</p>
<p>goreman 是一个 Go 语言编写的多进程管理工具，是对 Ruby 下广泛使用的 Foreman 的重写。下面我们使用 goreman 来演示在一台物理机上面以静态方式启动 etcd 集群。</p>
<ul>
<li>前面我们已经确认过 Go 语言的安装环境，现在可以直接执行：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">go get github.com&#x2F;mattn&#x2F;goreman</span><br></pre></td></tr></table></figure>

<blockquote>
<p>编译后的文件放在<code>$GOPATH/bin</code>中，<code>$GOPATH/bin</code>目录已经添加到了系统<code>$PATH</code>中，所以我们可以方便地执行命令<code>goreman</code>命令。</p>
</blockquote>
<ul>
<li>下面就是编写 Procfile 脚本，我们启动三个 etcd，具体对应如下：</li>
</ul>
<table>
<thead>
<tr>
<th>HostName</th>
<th>IP</th>
<th>客户端交互端口</th>
<th>peer通信端口</th>
</tr>
</thead>
<tbody><tr>
<td>infra1</td>
<td>127.0.0.1</td>
<td>12379</td>
<td>12380</td>
</tr>
<tr>
<td>infra2</td>
<td>127.0.0.1</td>
<td>22379</td>
<td>22380</td>
</tr>
<tr>
<td>infra3</td>
<td>127.0.0.1</td>
<td>32379</td>
<td>32380</td>
</tr>
</tbody></table>
<ul>
<li>Procfile 脚本中 infra1 启动命令如下：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">etcd --name infra1 --listen-client-urls http:&#x2F;&#x2F;127.0.0.1:12379 --advertise-client-urls http:&#x2F;&#x2F;127.0.0.1:12379 --listen-peer-urls http:&#x2F;&#x2F;127.0.0.1:12380 --initial-advertise-peer-urls http:&#x2F;&#x2F;127.0.0.1:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster &#39;infra1&#x3D;http:&#x2F;&#x2F;127.0.0.1:12380,infra2&#x3D;http:&#x2F;&#x2F;127.0.0.1:22380,infra3&#x3D;http:&#x2F;&#x2F;127.0.0.1:32380&#39; --initial-cluster-state new --enable-pprof --logger&#x3D;zap --log-outputs&#x3D;stderr</span><br></pre></td></tr></table></figure>

<p>infra2 和 infra3 的启动命令类似。</p>
<p>下面我们看一下其中各配置项的说明。</p>
<ul>
<li><p>–name：etcd 集群中的节点名，这里可以随意，方便区分且不重复即可。</p>
</li>
<li><p>–listen-client-urls：监听用于客户端通信的 url，同样可以监听多个。</p>
</li>
<li><p>–advertise-client-urls：建议使用的客户端通信 url，该值用于 etcd 代理或 etcd 成员与 etcd 节点通信。</p>
</li>
<li><p>–listen-peer-urls：监听用于节点之间通信的 url，可监听多个，集群内部将通过这些 url 进行数据交互 (如选举、数据同步等)。</p>
</li>
<li><p>–initial-advertise-peer-urls：建议用于节点之间通信的 url，节点间将以该值进行通信。</p>
</li>
<li><p>–initial-cluster-token： etcd-cluster-1，节点的 token 值，设置该值后集群将生成唯一 ID，并为每个节点也生成唯一 ID。当使用相同配置文件再启动一个集群时，只要该 token 值不一样，etcd 集群就不会相互影响。</p>
</li>
<li><p>–initial-cluster：集群中所有的 initial-advertise-peer-urls 的合集。</p>
</li>
<li><p>–initial-cluster-state：new，新建集群的标志。</p>
</li>
</ul>
<p>注意上面的脚本，<strong>etcd 命令执行时需要根据本地实际的安装地址进行配置</strong>。</p>
<ul>
<li>下面我们使用<code>goreman</code>命令启动 etcd 集群：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">goreman -f &#x2F;opt&#x2F;procfile start</span><br></pre></td></tr></table></figure>



<ul>
<li>启动完成后查看集群内的成员：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">$</span><span class="bash"> etcdctl --endpoints=http://localhost:22379  member list</span></span><br><span class="line"></span><br><span class="line">8211f1d0f64f3269, started, infra1, http://127.0.0.1:12380, http://127.0.0.1:12379, false</span><br><span class="line"></span><br><span class="line">91bc3c398fb3c146, started, infra2, http://127.0.0.1:22380, http://127.0.0.1:22379, false</span><br><span class="line"></span><br><span class="line">fd422379fda50e48, started, infra3, http://127.0.0.1:32380, http://127.0.0.1:32379, false</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>现在我们的 etcd 集群已经搭建成功。需要注意的是：<strong>在集群启动时，我们通过静态的方式指定集群的成员。但在实际环境中，集群成员的 IP 可能不会提前知道，此时需要采用动态发现的机制</strong>。</p>
<h2 id="2-2-动态发现启动-etcd-集群"><a href="#2-2-动态发现启动-etcd-集群" class="headerlink" title="2.2 动态发现启动 etcd 集群"></a>2.2 动态发现启动 etcd 集群</h2><p>Discovery Service，即发现服务，帮助新的 etcd 成员使用共享 URL 在集群引导阶段发现所有其他成员。</p>
<p>Discovery Service 使用已有的 etcd 集群来协调新集群的启动，主要操作如下：</p>
<ul>
<li><p>首先，所有新成员都与发现服务交互，并帮助生成预期的成员列表；</p>
</li>
<li><p>之后，每个新成员使用此列表引导其服务器，该列表执行与<code>--initial-cluster</code>标志相同的功能，即设置所有集群的成员信息。</p>
</li>
</ul>
<p>我们的实验中启动三个 etcd 实例，具体对应如下：</p>
<table>
<thead>
<tr>
<th>HostName</th>
<th>IP</th>
<th>客户端交互端口</th>
<th>peer通信端口</th>
</tr>
</thead>
<tbody><tr>
<td>etcd1</td>
<td>192.168.202.128</td>
<td>2379</td>
<td>2280</td>
</tr>
<tr>
<td>etcd2</td>
<td>192.168.202.129</td>
<td>2379</td>
<td>2280</td>
</tr>
<tr>
<td>etcd3</td>
<td>192.168.202.130</td>
<td>2379</td>
<td>2280</td>
</tr>
</tbody></table>
<p>下面就开始以动态发现的方式来启动 etcd 集群，具体步骤如下。</p>
<ul>
<li><strong>获取 discovery 的 token</strong></li>
</ul>
<p><strong>首先需要生成标识新集群的唯一令牌</strong>。该令牌将用于键空间中的唯一前缀，比较简单的方法是使用 uuidgen 生成 UUID：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">UUID&#x3D;$(uuidgen)</span><br></pre></td></tr></table></figure>



<ul>
<li><strong>指定集群的大小</strong></li>
</ul>
<p>获取令牌时，必须指定群集大小。<strong>发现服务使用该数值来确定组成集群的所有成员</strong>：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -X PUT http:&#x2F;&#x2F;10.0.10.10:2379&#x2F;v2&#x2F;keys&#x2F;discovery&#x2F;6c007a14875d53d9bf0ef5a6fc0257c817f0fb83&#x2F;_config&#x2F;size -d value&#x3D;3</span><br></pre></td></tr></table></figure>

<p>我们需要把该 url 作为<code>--discovery</code>参数来启动 etcd，节点会自动使用该路径对应的目录进行 etcd 的服务注册与发现。</p>
<ul>
<li><strong>公共发现服务</strong></li>
</ul>
<p>当我们本地没有可用的 etcd 集群，etcd 官网提供了一个可以公网访问的 etcd 存储地址。你可以通过如下命令得到 etcd 服务的目录，并把它作为<code>--discovery</code>参数使用。</p>
<p>公共发现服务<code>discovery.etcd.io</code>以相同的方式工作，但是有一层修饰，在此之上仍然使用 etcd 群集作为数据存储。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ curl https:&#x2F;&#x2F;discovery.etcd.io&#x2F;new?size&#x3D;3</span><br><span class="line"></span><br><span class="line">https:&#x2F;&#x2F;discovery.etcd.io&#x2F;3e86b59982e49066c5d813af1c2e2579cbf573de</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li><strong>以动态发现方式启动集群</strong></li>
</ul>
<p>etcd 发现模式下，启动 etcd 的命令如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"># etcd1 启动</span><br><span class="line"></span><br><span class="line">$ &#x2F;opt&#x2F;etcd&#x2F;bin&#x2F;etcd  --name etcd1 --initial-advertise-peer-urls http:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">  --listen-peer-urls http:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">  --data-dir &#x2F;opt&#x2F;etcd&#x2F;data \</span><br><span class="line"></span><br><span class="line">  --listen-client-urls http:&#x2F;&#x2F;192.168.202.128:2379,http:&#x2F;&#x2F;127.0.0.1:2379 \</span><br><span class="line"></span><br><span class="line">  --advertise-client-urls http:&#x2F;&#x2F;192.168.202.128:2379 \</span><br><span class="line"></span><br><span class="line">  --discovery https:&#x2F;&#x2F;discovery.etcd.io&#x2F;3e86b59982e49066c5d813af1c2e2579cbf573de</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>etcd2 和 etcd3 启动类似，替换 listen-peer-urls 和 advertise-client-urls 即可。需要注意的是：<strong>在我们完成了集群的初始化后，</strong><code>--discovery</code>就失去了作用。当需要增加节点时，需要使用 etcdctl 进行操作。</p>
<p>为了安全，每次启动新 etcd 集群时，都会使用新的 discovery token 进行注册。当出现初始化启动的节点超过了指定的数量时，多余的节点会自动转化为 Proxy 模式的 etcd（在后面课时会详细介绍）。</p>
<ul>
<li><strong>结果验证</strong></li>
</ul>
<p>集群启动后，进行验证，我们看一下集群节点的健康状态：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ &#x2F;opt&#x2F;etcd&#x2F;bin&#x2F;etcdctl  --endpoints&#x3D;&quot;http:&#x2F;&#x2F;192.168.202.128:2379,http:&#x2F;&#x2F;192.168.202.129:2379,http:&#x2F;&#x2F;192.168.202.130:2379&quot;  endpoint  health</span><br><span class="line"></span><br><span class="line"># 结果如下</span><br><span class="line"></span><br><span class="line">    http:&#x2F;&#x2F;192.168.202.128:2379 is healthy: successfully committed proposal: took &#x3D; 3.157068ms</span><br><span class="line"></span><br><span class="line">    http:&#x2F;&#x2F;192.168.202.130:2379 is healthy: successfully committed proposal: took &#x3D; 3.300984ms</span><br><span class="line"></span><br><span class="line">    http:&#x2F;&#x2F;192.168.202.129:2379 is healthy: successfully committed proposal: took &#x3D; 3.263923ms</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>可以看到，集群中的三个节点都是健康的正常状态，可以说明以动态发现方式启动集群成功。</p>
<p>除此之外，<strong>etcd 还支持使用 DNS SRV 记录启动 etcd 集群</strong>。使用 DNSmasq 创建 DNS 服务，实际上是利用 DNS 的 SRV 记录不断轮训查询实现，DNS SRV 是 DNS 数据库中支持的一种资源记录的类型，它记录了计算机与所提供服务信息的对应关系。</p>
<h1 id="3-etcd-通信安全"><a href="#3-etcd-通信安全" class="headerlink" title="3 etcd 通信安全"></a>3 etcd 通信安全</h1><p>etcd 支持通过 TLS 协议进行的加密通信，TLS 通道可用于对等体（指 etcd 集群中的服务实例）之间的加密内部群集通信以及加密的客户端流量。这一节我们看一下客户端 TLS 设置群集的实现。</p>
<p>想要实现数据 HTTPS 加密协议访问、保障数据的安全，就需要 SSL 证书，TLS 是 SSL 与 HTTPS 安全传输层协议名称。</p>
<h2 id="3-1-进行-TLS-加密实践"><a href="#3-1-进行-TLS-加密实践" class="headerlink" title="3.1 进行 TLS 加密实践"></a>3.1 进行 TLS 加密实践</h2><p>为了进行实践，我们将安装一些实用的命令行工具，这包括 CFSSL 、cfssljson。</p>
<p>CFSSL 是 CloudFlare 的 PKI/TLS 加密利器。它既是命令行工具，也可以用于签名、验证和捆绑 TLS 证书的 HTTP API 服务器，需要 Go 1.12+ 版本才能构建。</p>
<h3 id="3-1-1-环境配置"><a href="#3-1-1-环境配置" class="headerlink" title="3.1.1 环境配置"></a>3.1.1 <strong>环境配置</strong></h3><table>
<thead>
<tr>
<th>HostName</th>
<th>IP</th>
<th>客户端交互端口</th>
<th>peer通信端口</th>
</tr>
</thead>
<tbody><tr>
<td>infra0</td>
<td>192.168.202.128</td>
<td>2379</td>
<td>2280</td>
</tr>
<tr>
<td>infra1</td>
<td>192.168.202.129</td>
<td>2379</td>
<td>2280</td>
</tr>
<tr>
<td>infra2</td>
<td>192.168.202.130</td>
<td>2379</td>
<td>2280</td>
</tr>
</tbody></table>
<h3 id="3-1-2-安装-CFSSL"><a href="#3-1-2-安装-CFSSL" class="headerlink" title="3.1.2 安装 CFSSL"></a>3.1.2 <strong>安装 CFSSL</strong></h3><ul>
<li>安装CFSSL</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">$ ls ~&#x2F;Downloads&#x2F;cfssl</span><br><span class="line"></span><br><span class="line">cfssl-certinfo_1.4.1_linux_amd64 cfssl_1.4.1_linux_amd64          cfssljson_1.4.1_linux_amd64</span><br><span class="line"></span><br><span class="line">chmod +x cfssl_1.4.1_linux_amd64 cfssljson_1.4.1_linux_amd64 cfssl-certinfo_1.4.1_linux_amd64</span><br><span class="line"></span><br><span class="line">mv cfssl_1.4.1_linux_amd64 &#x2F;usr&#x2F;local&#x2F;bin&#x2F;cfssl</span><br><span class="line"></span><br><span class="line">mv cfssljson_1.4.1_linux_amd64 &#x2F;usr&#x2F;local&#x2F;bin&#x2F;cfssljson</span><br><span class="line"></span><br><span class="line">mv cfssl-certinfo_1.4.1_linux_amd64 &#x2F;usr&#x2F;bin&#x2F;cfssl-certinfo</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>安装完成后，查看版本信息如下所示：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$ cfssl version</span><br><span class="line"></span><br><span class="line">Version: 1.4.1</span><br><span class="line"></span><br><span class="line">Runtime: go1.12.12</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="3-1-3-配置-CA-并创建-TLS-证书"><a href="#3-1-3-配置-CA-并创建-TLS-证书" class="headerlink" title="3.1.3 配置 CA 并创建 TLS 证书"></a>3.1.3 <strong>配置 CA 并创建 TLS 证书</strong></h3><p>我们将使用 CloudFlare’s PKI 工具 CFSSL 来配置 PKI 安全策略，然后用它创建 Certificate Authority（CA，即证书机构），并为 etcd 创建 TLS 证书。</p>
<ul>
<li>首先创建 SSL 配置目录：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">mkdir &#x2F;opt&#x2F;etcd&#x2F;&#123;bin,cfg,ssl&#125; -p</span><br><span class="line"></span><br><span class="line">cd &#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;</span><br></pre></td></tr></table></figure>



<ul>
<li>接下来完善 etcd CA 配置，写入 ca-config.json 如下的配置：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt; EOF | tee ca-config.json</span><br><span class="line"></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">  &quot;signing&quot;: &#123;</span><br><span class="line"></span><br><span class="line">    &quot;default&quot;: &#123;</span><br><span class="line"></span><br><span class="line">      &quot;expiry&quot;: &quot;87600h&quot;</span><br><span class="line"></span><br><span class="line">    &#125;,</span><br><span class="line"></span><br><span class="line">    &quot;profiles&quot;: &#123;</span><br><span class="line"></span><br><span class="line">      &quot;etcd&quot;: &#123;</span><br><span class="line"></span><br><span class="line">         &quot;expiry&quot;: &quot;87600h&quot;,</span><br><span class="line"></span><br><span class="line">         &quot;usages&quot;: [</span><br><span class="line"></span><br><span class="line">            &quot;signing&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;key encipherment&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;server auth&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;client auth&quot;</span><br><span class="line"></span><br><span class="line">        ]</span><br><span class="line"></span><br><span class="line">      &#125;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">EOF</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>生成获取 etcd ca 证书，需要证书签名的请求文件，因此在 ca-csr.json 写入如下的配置：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt; EOF | tee ca-csr.json</span><br><span class="line"></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">    &quot;CN&quot;: &quot;etcd CA&quot;,</span><br><span class="line"></span><br><span class="line">    &quot;key&quot;: &#123;</span><br><span class="line"></span><br><span class="line">        &quot;algo&quot;: &quot;rsa&quot;,</span><br><span class="line"></span><br><span class="line">        &quot;size&quot;: 2048</span><br><span class="line"></span><br><span class="line">    &#125;,</span><br><span class="line"></span><br><span class="line">    &quot;names&quot;: [</span><br><span class="line"></span><br><span class="line">        &#123;</span><br><span class="line"></span><br><span class="line">            &quot;C&quot;: &quot;CN&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;L&quot;: &quot;Shanghai&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;ST&quot;: &quot;Shanghai&quot;</span><br><span class="line"></span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    ]</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">EOF</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>根据上面的配置，生成 CA 凭证和私钥：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">$ cfssl gencert -initca ca-csr.json | cfssljson -bare ca</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>生成 etcd server 证书，写入 server-csr.json 如下的配置：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt; EOF | tee server-csr.json</span><br><span class="line"></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">    &quot;CN&quot;: &quot;etcd&quot;,</span><br><span class="line"></span><br><span class="line">    &quot;hosts&quot;: [</span><br><span class="line"></span><br><span class="line">    &quot;192.168.202.128&quot;,</span><br><span class="line"></span><br><span class="line">    &quot;192.168.202.129&quot;,</span><br><span class="line"></span><br><span class="line">    &quot;192.168.202.130&quot;</span><br><span class="line"></span><br><span class="line">    ],</span><br><span class="line"></span><br><span class="line">    &quot;key&quot;: &#123;</span><br><span class="line"></span><br><span class="line">        &quot;algo&quot;: &quot;rsa&quot;,</span><br><span class="line"></span><br><span class="line">        &quot;size&quot;: 2048</span><br><span class="line"></span><br><span class="line">    &#125;,</span><br><span class="line"></span><br><span class="line">    &quot;names&quot;: [</span><br><span class="line"></span><br><span class="line">        &#123;</span><br><span class="line"></span><br><span class="line">            &quot;C&quot;: &quot;CN&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;L&quot;: &quot;Beijing&quot;,</span><br><span class="line"></span><br><span class="line">            &quot;ST&quot;: &quot;Beijing&quot;</span><br><span class="line"></span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    ]</span><br><span class="line"></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">EOF</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>最后就可以生成 server 证书了：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cfssl gencert -ca&#x3D;ca.pem -ca-key&#x3D;ca-key.pem -config&#x3D;ca-config.json -profile&#x3D;etcd server-csr.json | cfssljson -bare server</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<ul>
<li>启动 etcd 集群，配置如下：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">#etcd1 启动</span><br><span class="line"></span><br><span class="line">$ &#x2F;opt&#x2F;etcd&#x2F;bin&#x2F;etcd --name etcd1 --initial-advertise-peer-urls https:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">     --listen-peer-urls https:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">     --listen-client-urls https:&#x2F;&#x2F;192.168.202.128:2379,https:&#x2F;&#x2F;127.0.0.1:2379 \</span><br><span class="line"></span><br><span class="line">     --advertise-client-urls https:&#x2F;&#x2F;192.168.202.128:2379 \</span><br><span class="line"></span><br><span class="line">     --initial-cluster-token etcd-cluster-1 \</span><br><span class="line"></span><br><span class="line">     --initial-cluster etcd1&#x3D;https:&#x2F;&#x2F;192.168.202.128:2380, etcd2&#x3D;https:&#x2F;&#x2F;192.168.202.129:2380, etcd3&#x3D;https:&#x2F;&#x2F;192.168.202.130:2380 \</span><br><span class="line"></span><br><span class="line">     --initial-cluster-state new \</span><br><span class="line"></span><br><span class="line">     --client-cert-auth --trusted-ca-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;ca.pem \</span><br><span class="line"></span><br><span class="line">     --cert-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server.pem --key-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server-key.pem \</span><br><span class="line"></span><br><span class="line">     --peer-client-cert-auth --peer-trusted-ca-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;ca.pem \</span><br><span class="line"></span><br><span class="line">     --peer-cert-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server.pem --peer-key-file&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server-key.pem</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>etcd2 和 etcd3 启动类似，<strong>注意替换 listen-peer-urls 和 advertise-client-urls</strong>。通过三台服务器的控制台可以知道，集群已经成功建立。</p>
<ul>
<li>下面我们进行验证：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ &#x2F;opt&#x2F;etcd&#x2F;bin&#x2F;etcdctl --cacert&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;ca.pem --cert&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server.pem --key&#x3D;&#x2F;opt&#x2F;etcd&#x2F;ssl&#x2F;server-key.pem --endpoints&#x3D;&quot;https:&#x2F;&#x2F;192.168.202.128:2379,https:&#x2F;&#x2F;192.168.202.129:2379,https:&#x2F;&#x2F;192.168.202.130:2379&quot;  endpoint health</span><br><span class="line"></span><br><span class="line"># 输出如下：</span><br><span class="line"></span><br><span class="line">https:&#x2F;&#x2F;192.168.202.129:2379 is healthy: successfully committed proposal: took &#x3D; 9.492956ms</span><br><span class="line"></span><br><span class="line">https:&#x2F;&#x2F;192.168.202.130:2379 is healthy: successfully committed proposal: took &#x3D; 12.805109ms</span><br><span class="line"></span><br><span class="line">https:&#x2F;&#x2F;192.168.202.128:2379 is healthy: successfully committed proposal: took &#x3D; 13.036091ms</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>查看三个节点的健康状况，<code>endpoint health</code>，输出的结果符合我们的预期。</p>
<p>经过 TLS 加密的 etcd 集群，在进行操作时，需要加上认证相关的信息。</p>
<h3 id="3-1-4-自动证书"><a href="#3-1-4-自动证书" class="headerlink" title="3.1.4 自动证书"></a>3.1.4 <strong>自动证书</strong></h3><p>如果集群需要加密的通信但不需要经过身份验证的连接，可以将 etcd 配置为自动生成其密钥。在初始化时，每个成员都基于其通告的 IP 地址和主机创建自己的密钥集。</p>
<p>在每台机器上，etcd 将使用以下标志启动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">$ etcd --name etcd1 --initial-advertise-peer-urls https:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">  --listen-peer-urls https:&#x2F;&#x2F;192.168.202.128:2380 \</span><br><span class="line"></span><br><span class="line">  --listen-client-urls https:&#x2F;&#x2F;192.168.202.128:2379,https:&#x2F;&#x2F;127.0.0.1:2379 \</span><br><span class="line"></span><br><span class="line">  --advertise-client-urls https:&#x2F;&#x2F;10.0.1.10:2379 \</span><br><span class="line"></span><br><span class="line">  --initial-cluster-token etcd-cluster-1 \</span><br><span class="line"></span><br><span class="line">  --initial-cluster infra0&#x3D;https:&#x2F;&#x2F;192.168.202.128:2380,infra1&#x3D;https:&#x2F;&#x2F;192.168.202.129:2380,infra2&#x3D;https:&#x2F;&#x2F;192.168.202.130:2380 \</span><br><span class="line"></span><br><span class="line">  --initial-cluster-state new \</span><br><span class="line"></span><br><span class="line">  --auto-tls \</span><br><span class="line"></span><br><span class="line">  --peer-auto-tls</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>由于自动签发证书并不能认证身份，直接 curl 会返回错误，<strong>需要使用 curl 的<code>-k</code>命令屏蔽对证书链的校验</strong>。</p>
<p>Discovery Service 用于生成集群的发现令牌，需要注意的是，<strong>该令牌仅用于集群引导阶段，不能用于运行时重新配置或集群监视</strong>。一个发现令牌只能代表一个 etcd 集群，只要此令牌上的发现协议启动，即使它中途失败，也不能用于引导另一个 etcd 集群。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%20%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2%EF%BC%9A%E5%8D%95%E6%9C%BA%E5%92%8C%E9%9B%86%E7%BE%A4/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/Kubernetes知识拓展-默认调度器执行逻辑" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/Kubernetes%E7%9F%A5%E8%AF%86%E6%8B%93%E5%B1%95-%E9%BB%98%E8%AE%A4%E8%B0%83%E5%BA%A6%E5%99%A8%E6%89%A7%E8%A1%8C%E9%80%BB%E8%BE%91/">k8s知识拓展默认调度器策略</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/Kubernetes%E7%9F%A5%E8%AF%86%E6%8B%93%E5%B1%95-%E9%BB%98%E8%AE%A4%E8%B0%83%E5%BA%A6%E5%99%A8%E6%89%A7%E8%A1%8C%E9%80%BB%E8%BE%91/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="Kubernetes默认调度器"><a href="#Kubernetes默认调度器" class="headerlink" title="Kubernetes默认调度器"></a>Kubernetes默认调度器</h1><h2 id="1-调度流程"><a href="#1-调度流程" class="headerlink" title="1 调度流程"></a>1 调度流程</h2><p>在 Kubernetes 项目中，默认调度器的主要职责，就是为一个新创建出来的 Pod，寻找一个最合适的节点（Node）。</p>
<p>最合适”的含义，包括两层：</p>
<ol>
<li>从集群所有的节点中，根据调度算法挑选出所有可以运行该 Pod 的节点；</li>
<li>从第一步的结果中，再根据调度算法挑选一个最符合条件的节点作为最终结果。</li>
</ol>
<p>在具体的调度流程中，</p>
<ol>
<li>默认调度器会首先调用一组叫作 Predicate 的调度算法，来检查每个 Node。</li>
<li>然后，再调用一组叫作 Priority 的调度算法，来给上一步得到的结果里的每个 Node 打分。</li>
<li>最终的调度结果，就是得分最高的那个 Node。</li>
<li>调度器对一个 Pod 调度成功，实际上就是将它的 spec.nodeName 字段填上调度结果的节点名字。</li>
</ol>
<p>上述调度机制的工作原理，可以用如下所示的一幅示意图来表示</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210426140506917.png" alt="image-20210426140506917"></p>
<h2 id="2-控制循环"><a href="#2-控制循环" class="headerlink" title="2 控制循环"></a>2 控制循环</h2><p>从上面的图中得知: Kubernetes 的调度器的核心，实际上就是两个相互独立的控制循环。</p>
<h3 id="2-1-Informer-Path"><a href="#2-1-Informer-Path" class="headerlink" title="2.1 Informer Path"></a>2.1 Informer Path</h3><p>第一个控制循环，我们可以称之为 Informer Path。</p>
<p>它的主要目的，是启动一系列 Informer，用来监听（Watch）Etcd 中 Pod、Node、Service 等与调度相关的 API 对象的变化。</p>
<blockquote>
<p>比如，当一个待调度 Pod（即：它的 nodeName 字段是空的）被创建出来之后，调度器就会通过 Pod Informer 的 Handler，将这个待调度 Pod 添加进调度队列。</p>
</blockquote>
<p>在默认情况下，Kubernetes 的调度队列是一个 PriorityQueue（优先级队列），并且当某些集群信息发生变化的时候，调度器还会对调度队列里的内容进行一些特殊操作。</p>
<p>此外，Kubernetes 的默认调度器还要负责对调度器缓存（即：scheduler cache）进行更新。事实上，Kubernetes 调度部分进行性能优化的一个<strong>最根本原则，就是尽最大可能将集群信息 Cache 化，以便从根本上提高 Predicate 和 Priority 调度算法的执行效率。</strong></p>
<h3 id="2-2-Scheduling-Path"><a href="#2-2-Scheduling-Path" class="headerlink" title="2.2 Scheduling Path"></a>2.2 Scheduling Path</h3><p>第二个控制循环，是调度器负责 Pod 调度的主循环，我们可以称之为 Scheduling Path。</p>
<p>Scheduling Path 的主要逻辑，就是不断地从调度队列里出队一个 Pod。</p>
<p>然后，调用 Predicates 算法进行“过滤”。这一步“过滤”得到的一组 Node，就是所有可以运行这个 Pod 的宿主机列表。当然，Predicates 算法需要的 Node 信息，都是从 Scheduler Cache 里直接拿到的，这是调度器保证算法执行效率的主要手段之一。</p>
<p>接下来，调度器就会再调用 Priorities 算法为上述列表里的 Node 打分，分数从 0 到 10。得分最高的 Node，就会作为这次调度的结果。</p>
<p>调度算法执行完成后，调度器就需要将 Pod 对象的 nodeName 字段的值，修改为上述 Node 的名字。<strong>这个步骤在 Kubernetes 里面被称作 Bind</strong>。</p>
<h3 id="2-3-Assume-amp-Admit"><a href="#2-3-Assume-amp-Admit" class="headerlink" title="2.3 Assume &amp; Admit"></a>2.3 Assume &amp; Admit</h3><p>为了不在关键调度路径里远程访问 APIServer，Kubernetes 的默认调度器在 Bind 阶段，只会更新 Scheduler Cache 里的 Pod 和 Node 的信息。<strong>这种基于“乐观”假设的 API 对象更新方式，在 Kubernetes 里被称作 Assume。</strong></p>
<p>Assume 之后，调度器才会创建一个 Goroutine 来异步地向 APIServer 发起更新 Pod 的请求，来真正完成 Bind 操作。如果这次异步的 Bind 过程失败了，其实也没有太大关系，等 Scheduler Cache 同步之后一切就会恢复正常。</p>
<p>当然，正是由于上述 Kubernetes 调度器的“乐观”绑定的设计，当一个新的 Pod 完成调度需要在某个节点上运行起来之前，该节点上的 kubelet 还会通过一个叫作 Admit 的操作来再次验证该 Pod 是否确实能够运行在该节点上。这一步 Admit 操作，实际上就是把一组叫作 GeneralPredicates 的、最基本的调度算法，比如：“资源是否可用”“端口是否冲突”等再执行一遍，作为 kubelet 端的二次确认。</p>
<h3 id="2-4-Cache-化、乐观绑定、无锁化"><a href="#2-4-Cache-化、乐观绑定、无锁化" class="headerlink" title="2.4 Cache 化、乐观绑定、无锁化"></a>2.4 Cache 化、乐观绑定、无锁化</h3><p><strong>除了上面讲到的的“Cache 化”和“乐观绑定”，Kubernetes 默认调度器还有一个重要的设计，那就是“无锁化”</strong></p>
<p>在 Scheduling Path 上，调度器会启动多个 Goroutine 以节点为粒度并发执行 Predicates 算法，从而提高这一阶段的执行效率。而与之类似的，Priorities 算法也会以 MapReduce 的方式并行计算然后再进行汇总。而在这些所有需要并发的路径上，调度器会避免设置任何全局的竞争资源，从而免去了使用锁进行同步带来的巨大的性能损耗。</p>
<p>所以，在这种思想的指导下，如果你再去查看一下前面的调度器原理图，你就会发现，Kubernetes 调度器只有对调度队列和 Scheduler Cache 进行操作时，才需要加锁。而这两部分操作，都不在 Scheduling Path 的算法执行路径上。</p>
<p>当然，Kubernetes 调度器的上述设计思想，也是在集群规模不断增长的演进过程中逐步实现的。<strong>尤其是 “Cache 化”，这个变化其实是最近几年 Kubernetes 调度器性能得以提升的一个关键演化</strong>。</p>
<h1 id="Kubernetes默认调度器调度策略解析"><a href="#Kubernetes默认调度器调度策略解析" class="headerlink" title="Kubernetes默认调度器调度策略解析"></a>Kubernetes默认调度器调度策略解析</h1><h2 id="1-Predicates"><a href="#1-Predicates" class="headerlink" title="1 Predicates"></a>1 Predicates</h2><p><strong>Predicates 在调度过程中的作用，可以理解为 Filter</strong>，即：它按照调度策略，从当前集群的所有节点中，“过滤”出一系列符合条件的节点。这些节点，都是可以运行待调度 Pod 的宿主机。</p>
<p>而在 Kubernetes 中，默认的调度策略有如下三种：</p>
<h3 id="1-1-GeneralPredicates"><a href="#1-1-GeneralPredicates" class="headerlink" title="1.1 GeneralPredicates"></a>1.1 GeneralPredicates</h3><p>第一种类型，叫作 GeneralPredicates。</p>
<p>顾名思义，这一组过滤规则，负责的是最基础的调度策略。比如，PodFitsResources 计算的就是宿主机的 CPU 和内存资源等是否够用。</p>
<p>比如，PodFitsResources 计算的就是宿主机的 CPU 和内存资源等是否够用。</p>
<p>而 PodFitsHost 检查的是，宿主机的名字是否跟 Pod 的 spec.nodeName 一致。</p>
<p>PodFitsHostPorts 检查的是，Pod 申请的宿主机端口（spec.nodePort）是不是跟已经被使用的端口有冲突。</p>
<p>PodMatchNodeSelector 检查的是，Pod 的 nodeSelector 或者 nodeAffinity 指定的节点，是否与待考察节点匹配，等等。</p>
<p>可以看到，像上面这样一组 GeneralPredicates，正是 Kubernetes 考察一个 Pod 能不能运行在一个 Node 上最基本的过滤条件。所以，GeneralPredicates 也会被其他组件（比如 kubelet）直接调用。</p>
<h3 id="1-2-Volume-过滤规则"><a href="#1-2-Volume-过滤规则" class="headerlink" title="1.2 Volume 过滤规则"></a>1.2 Volume 过滤规则</h3><p>第二种类型，是与 Volume 相关的过滤规则。</p>
<p>这一组过滤规则，负责的是跟容器持久化 Volume 相关的调度策略。</p>
<p>其中，NoDiskConflict 检查的条件，是多个 Pod 声明挂载的持久化 Volume 是否有冲突。</p>
<p>而 MaxPDVolumeCountPredicate 检查的条件，则是一个节点上某种类型的持久化 Volume 是不是已经超过了一定数目，如果是的话，那么声明使用该类型持久化 Volume 的 Pod 就不能再调度到这个节点了。</p>
<p>而 VolumeZonePredicate，则是检查持久化 Volume 的 Zone（高可用域）标签，是否与待考察节点的 Zone 标签相匹配。</p>
<p>此外，这里还有一个叫作 VolumeBindingPredicate 的规则。它负责检查的，是该 Pod 对应的 PV 的 nodeAffinity 字段，是否跟某个节点的标签相匹配。</p>
<h3 id="1-3-宿主机过滤规则"><a href="#1-3-宿主机过滤规则" class="headerlink" title="1.3 宿主机过滤规则"></a>1.3 宿主机过滤规则</h3><p>第三种类型，是宿主机相关的过滤规则。</p>
<p>这一组规则，主要考察待调度 Pod 是否满足 Node 本身的某些条件。</p>
<p>比如，PodToleratesNodeTaints，负责检查的就是我们前面经常用到的 Node 的“污点”机制。只有当 Pod 的 Toleration 字段与 Node 的 Taint 字段能够匹配的时候，这个 Pod 才能被调度到该节点上。</p>
<p>而 NodeMemoryPressurePredicate，检查的是当前节点的内存是不是已经不够充足，如果是的话，那么待调度 Pod 就不能被调度到该节点上。</p>
<h3 id="1-4-Pod-过滤规则"><a href="#1-4-Pod-过滤规则" class="headerlink" title="1.4 Pod 过滤规则"></a>1.4 Pod 过滤规则</h3><p>第四种类型，是 Pod 相关的过滤规则。</p>
<p>这一组规则，跟 GeneralPredicates 大多数是重合的。而比较特殊的，是 PodAffinityPredicate。这个规则的作用，是检查待调度 Pod 与 Node 上的已有 Pod 之间的亲密（affinity）和反亲密（anti-affinity）关系。</p>
<h3 id="1-5-总结"><a href="#1-5-总结" class="headerlink" title="1.5 总结"></a>1.5 总结</h3><p><strong>在具体执行的时候， 当开始调度一个 Pod 时，Kubernetes 调度器会同时启动 16 个 Goroutine，来并发地为集群里的所有 Node 计算 Predicates，最后返回可以运行这个 Pod 的宿主机列表。</strong></p>
<p>需要注意的是，在为每个 Node 执行 Predicates 时，调度器会按照固定的顺序来进行检查。这个顺序，是按照 Predicates 本身的含义来确定的。比如，宿主机相关的 Predicates 会被放在相对靠前的位置进行检查。要不然的话，在一台资源已经严重不足的宿主机上，上来就开始计算 PodAffinityPredicate，是没有实际意义的。</p>
<h2 id="2-Priorities"><a href="#2-Priorities" class="headerlink" title="2 Priorities"></a>2 Priorities</h2><p>在 Predicates 阶段完成了节点的“过滤”之后，Priorities 阶段的工作就是为这些节点打分。这里打分的范围是 0-10 分，得分最高的节点就是最后被 Pod 绑定的最佳节点。</p>
<h3 id="2-1-LeastRequestedPriority"><a href="#2-1-LeastRequestedPriority" class="headerlink" title="2.1 LeastRequestedPriority"></a>2.1 LeastRequestedPriority</h3><p>Priorities 里最常用到的一个打分规则，是 LeastRequestedPriority。它的计算方法，可以简单地总结为如下所示的公式：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">score &#x3D; (cpu((capacity-sum(requested))10&#x2F;capacity) + memory((capacity-sum(requested))10&#x2F;capacity))&#x2F;2</span><br></pre></td></tr></table></figure>

<blockquote>
<p>可以看到，这个算法实际上就是在选择空闲资源（CPU 和 Memory）最多的宿主机。</p>
</blockquote>
<h3 id="2-2-BalancedResourceAllocation"><a href="#2-2-BalancedResourceAllocation" class="headerlink" title="2.2 BalancedResourceAllocation"></a>2.2 BalancedResourceAllocation</h3><p>而与 LeastRequestedPriority 一起发挥作用的，还有 BalancedResourceAllocation。它的计算公式如下所示：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">score &#x3D; 10 - variance(cpuFraction,memoryFraction,volumeFraction)*10</span><br></pre></td></tr></table></figure>

<p>其中，每种资源的 Fraction 的定义是 ：Pod 请求的资源 / 节点上的可用资源。而 variance 算法的作用，则是计算每两种资源 Fraction 之间的“距离”。而最后选择的，则是资源 Fraction 差距最小的节点。</p>
<p>所以说，BalancedResourceAllocation 选择的，其实是调度完成后，所有节点里各种资源分配最均衡的那个节点，从而避免一个节点上 CPU 被大量分配、而 Memory 大量剩余的情况。</p>
<h3 id="2-3-ImageLocalityPriority"><a href="#2-3-ImageLocalityPriority" class="headerlink" title="2.3 ImageLocalityPriority"></a>2.3 ImageLocalityPriority</h3><p>在默认 Priorities 里，还有一个叫作 ImageLocalityPriority 的策略。它是在 Kubernetes v1.12 里新开启的调度规则，即：如果待调度 Pod 需要使用的镜像很大，并且已经存在于某些 Node 上，那么这些 Node 的得分就会比较高。</p>
<p>当然，为了避免这个算法引发调度堆叠，调度器在计算得分的时候还会根据镜像的分布进行优化，即：如果大镜像分布的节点数目很少，那么这些节点的权重就会被调低，从而“对冲”掉引起调度堆叠的风险。</p>
<h3 id="2-4-其他的Priority"><a href="#2-4-其他的Priority" class="headerlink" title="2.4 其他的Priority"></a>2.4 其他的Priority</h3><p>此外，还有 <strong>NodeAffinityPriority、TaintTolerationPriority 和 InterPodAffinityPriority</strong> 这三种 Priority。顾名思义，它们与前面的 PodMatchNodeSelector、PodToleratesNodeTaints 和 PodAffinityPredicate 这三个 Predicate 的含义和计算方法是类似的。但是作为 Priority，一个 Node 满足上述规则的字段数目越多，它的得分就会越高。</p>
<h1 id="Kubernetes默认调度器的优先级与抢占机制"><a href="#Kubernetes默认调度器的优先级与抢占机制" class="headerlink" title="Kubernetes默认调度器的优先级与抢占机制"></a>Kubernetes默认调度器的优先级与抢占机制</h1><p>初了上面讲到的Kubernetes 默认调度器的主要调度算法的工作原理。在下面的内容中，再来讲解一下 Kubernetes 调度器里的另一个重要机制，即：优先级（Priority ）和抢占（Preemption）机制。</p>
<p><strong>首先需要明确的是，优先级和抢占机制，解决的是 Pod 调度失败时该怎么办的问题。</strong></p>
<h2 id="1-什么场景下需要用到优先级和抢占机制"><a href="#1-什么场景下需要用到优先级和抢占机制" class="headerlink" title="1 什么场景下需要用到优先级和抢占机制"></a>1 什么场景下需要用到优先级和抢占机制</h2><p>正常情况下，当一个 Pod 调度失败后，它就会被暂时“搁置”起来，直到 Pod 被更新，或者集群状态发生变化，调度器才会对这个 Pod 进行重新调度。</p>
<p>但在有时候，我们希望的是这样一个场景。当一个高优先级的 Pod 调度失败后，该 Pod 并不会被“搁置”，而是会“挤走”某个 Node 上的一些低优先级的 Pod 。这样就可以保证这个高优先级 Pod 的调度成功。这个特性，其实也是一直以来就存在于 Borg 以及 Mesos 等项目里的一个基本功能。</p>
<h2 id="2-PriorityClass"><a href="#2-PriorityClass" class="headerlink" title="2 PriorityClass"></a>2 PriorityClass</h2><p>在 Kubernetes 里，优先级和抢占机制是在 1.10 版本后才逐步可用的。要使用这个机制，你首先需要在 Kubernetes 里提交一个 PriorityClass 的定义，如下所示：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">scheduling.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">PriorityClass</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">high-priority</span></span><br><span class="line"><span class="attr">value:</span> <span class="number">1000000</span></span><br><span class="line"><span class="attr">globalDefault:</span> <span class="literal">false</span></span><br><span class="line"><span class="attr">description:</span> <span class="string">&quot;This priority class should be used for XYZ service pods only.&quot;</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p>上面这个 YAML 文件，定义的是一个名叫 high-priority 的 PriorityClass，其中 value 的值是 1000000 （一百万）。</p>
</blockquote>
<p><strong>Kubernetes 规定，优先级是一个 32 bit 的整数，最大值不超过 1000000000（10 亿，1 billion），并且值越大代表优先级越高。而超出 10 亿的值，其实是被 Kubernetes 保留下来分配给系统 Pod 使用的。</strong>显然，这样做的目的，就是保证系统 Pod 不会被用户抢占掉。</p>
<p>而一旦上述 YAML 文件里的 <code>globalDefault</code> 被设置为 true 的话，那就意味着这个 <code>PriorityClass</code> 的值会成为系统的默认值，系统中只能存在一个 <code>globalDefault</code> 设为true的<code>PriorityClass</code> 对象。。而如果这个值是 false，就表示我们只希望声明使用该 <code>PriorityClass</code> 的 Pod 拥有值为 1000000 的优先级，而对于没有声明 <code>PriorityClass </code>的 Pod 来说，它们的优先级就是 0。</p>
<p>在创建了 <code>PriorityClass </code>对象之后，Pod 就可以声明使用它了，如下所示：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: nginx</span><br><span class="line">  labels:</span><br><span class="line">    env: test</span><br><span class="line">spec:</span><br><span class="line">  containers:</span><br><span class="line">  - name: nginx</span><br><span class="line">    image: nginx</span><br><span class="line">    imagePullPolicy: IfNotPresent</span><br><span class="line">  priorityClassName: high-priority</span><br></pre></td></tr></table></figure>

<p>当 Pod 拥有了优先级之后，高优先级的 Pod 就可能会比低优先级的 Pod 提前出队，从而尽早完成调度过程。<strong>这个过程，就是“优先级”这个概念在 Kubernetes 里的主要体现。</strong></p>
<p>而当一个高优先级的 Pod 调度失败的时候，调度器的抢占能力就会被触发。这时，调度器就会试图从当前集群里寻找一个节点，使得当这个节点上的一个或者多个低优先级 Pod 被删除后，待调度的高优先级 Pod 就可以被调度到这个节点上。<strong>这个过程，就是“抢占”这个概念在 Kubernetes 里的主要体现。</strong></p>
<h2 id="3-抢占过程概述"><a href="#3-抢占过程概述" class="headerlink" title="3 抢占过程概述"></a>3 抢占过程概述</h2><p>为了方便叙述，我接下来会把待调度的高优先级 Pod 称为“抢占者”（Preemptor）。</p>
<p>当抢占过程发生时，抢占者并不会立刻被调度到被抢占的 Node 上。事实上，调度器只会将抢占者的 spec.nominatedNodeName 字段，设置为被抢占的 Node 的名字。然后，抢占者会重新进入下一个调度周期，然后在新的调度周期里来决定是不是要运行在被抢占的节点上。<strong>这当然也就意味着，即使在下一个调度周期，调度器也不会保证抢占者一定会运行在被抢占的节点上</strong>。</p>
<p>这样设计的一个重要原因是，调度器只会通过标准的 DELETE API 来删除被抢占的 Pod，所以，这些 Pod 必然是有一定的“优雅退出”时间（默认是 30s）的。而在这段时间里，其他的节点也是有可能变成可调度的，或者直接有新的节点被添加到这个集群中来。<strong>所以，鉴于优雅退出期间，集群的可调度性可能会发生的变化，把抢占者交给下一个调度周期再处理，是一个非常合理的选择</strong>。</p>
<p>而在抢占者等待被调度的过程中，如果有其他更高优先级的 Pod 也要抢占同一个节点，那么调度器就会清空原抢占者的 spec.nominatedNodeName 字段，从而允许更高优先级的抢占者执行抢占，并且，这也就使得原抢占者本身，也有机会去重新抢占其他节点。这些，都是设置 nominatedNodeName 字段的主要目的。</p>
<h2 id="4-调度队列"><a href="#4-调度队列" class="headerlink" title="4 调度队列"></a>4 调度队列</h2><p>Kubernetes 调度器实现抢占算法的一个最重要的设计，就是在调度队列的实现里，使用了两个不同的队列。</p>
<p><strong>第一个队列，叫作 activeQ</strong>。凡是在 activeQ 里的 Pod，都是下一个调度周期需要调度的对象。所以，当你在 Kubernetes 集群里新创建一个 Pod 的时候，调度器会将这个 Pod 入队到 activeQ 里面。而我在前面提到过的、调度器不断从队列里出队（Pop）一个 Pod 进行调度，实际上都是从 activeQ 里出队的。</p>
<p><strong>第二个队列，叫作 unschedulableQ</strong>，专门用来存放调度失败的 Pod。</p>
<p>而这里的一个关键点就在于，当一个 unschedulableQ 里的 Pod 被更新之后，调度器会自动把这个 Pod 移动到 activeQ 里，从而给这些调度失败的 Pod “重新做人”的机会。</p>
<p>然后，这次失败事件就会<strong>触发调度器为抢占者寻找牺牲者的流程</strong>。</p>
<h2 id="5-寻找牺牲者的流程"><a href="#5-寻找牺牲者的流程" class="headerlink" title="5 寻找牺牲者的流程"></a>5 寻找牺牲者的流程</h2><ul>
<li><p>第一步，调度器会检查这次失败事件的原因，来确认抢占是不是可以帮助抢占者找到一个新节点。这是因为有很多 Predicates 的失败是不能通过抢占来解决的。比如，PodFitsHost 算法（负责的是，检查 Pod 的 nodeSelector 与 Node 的名字是否匹配），这种情况下，除非 Node 的名字发生变化，否则你即使删除再多的 Pod，抢占者也不可能调度成功。</p>
</li>
<li><p>第二步，如果确定抢占可以发生，那么调度器就会把自己缓存的所有节点信息复制一份，然后使用这个副本来模拟抢占过程。</p>
</li>
</ul>
<p>这里的抢占过程很容易理解。调度器会检查缓存副本里的每一个节点，然后从该节点上最低优先级的 Pod 开始，逐一“删除”这些 Pod。而每删除一个低优先级 Pod，调度器都会检查一下抢占者是否能够运行在该 Node 上。一旦可以运行，调度器就记录下这个 Node 的名字和被删除 Pod 的列表，这就是一次抢占过程的结果了。</p>
<p>当遍历完所有的节点之后，调度器会在上述模拟产生的所有抢占结果里做一个选择，找出最佳结果。而这一步的<strong>判断原则，就是尽量减少抢占对整个系统的影响</strong>。比如，需要抢占的 Pod 越少越好，需要抢占的 Pod 的优先级越低越好，等等。</p>
<p>在得到了最佳的抢占结果之后，这个结果里的 Node，就是即将被抢占的 Node；被删除的 Pod 列表，就是牺牲者。所以接下来，<strong>调度器就可以真正开始抢占</strong>的操作了</p>
<h2 id="6-抢占操作"><a href="#6-抢占操作" class="headerlink" title="6 抢占操作"></a>6 抢占操作</h2><ol>
<li>第一步，调度器会检查牺牲者列表，清理这些 Pod 所携带的 nominatedNodeName 字段。</li>
<li>第二步，调度器会把抢占者的 nominatedNodeName，设置为被抢占的 Node 的名字。</li>
<li>第三步，调度器会开启一个 Goroutine，同步地删除牺牲者。</li>
</ol>
<p>而第二步对抢占者 Pod 的更新操作，就会触发到我前面提到的“重新做人”的流程，从而让抢占者在下一个调度周期重新进入调度流程。</p>
<p>所以接下来，<strong>调度器就会通过正常的调度流程把抢占者调度成功</strong>。这也是为什么，前面会说调度器并不保证抢占的结果：<strong>在这个正常的调度流程里，是一切皆有可能的</strong>。</p>
<h2 id="7-关于其他处于调度中的pod"><a href="#7-关于其他处于调度中的pod" class="headerlink" title="7 关于其他处于调度中的pod"></a>7 关于其他处于调度中的pod</h2><p>对于任意一个待调度 Pod 来说，因为有上述抢占者的存在，它的调度过程，其实是有一些特殊情况需要特殊处理的。</p>
<p>具体来说，在为某一对 Pod 和 Node 执行 Predicates 算法的时候，如果待检查的 Node 是一个即将被抢占的节点，即：调度队列里有 nominatedNodeName 字段值是该 Node 名字的 Pod 存在（可以称之为：“潜在的抢占者”）。<strong>那么，调度器就会对这个 Node ，将同样的 Predicates 算法运行两遍。</strong></p>
<ol>
<li><p><strong>第一遍</strong>， 调度器会假设上述“潜在的抢占者”已经运行在这个节点上，然后执行 Predicates 算法；</p>
</li>
<li><p><strong>第二遍</strong>， 调度器会正常执行 Predicates 算法，即：不考虑任何“潜在的抢占者”。</p>
</li>
</ol>
<p>而只有这两遍 Predicates 算法都能通过时，这个 Pod 和 Node 才会被认为是可以绑定（bind）的。</p>
<p>不难想到，这里需要执行第一遍 Predicates 算法的原因，是由于 InterPodAntiAffinity 规则的存在。</p>
<p>由于 InterPodAntiAffinity 规则关心待考察节点上所有 Pod 之间的互斥关系，所以我们在执行调度算法时必须考虑，如果抢占者已经存在于待考察 Node 上时，待调度 Pod 还能不能调度成功。</p>
<p>当然，这也就意味着，我们在这一步只需要考虑那些优先级等于或者大于待调度 Pod 的抢占者。毕竟对于其他较低优先级 Pod 来说，待调度 Pod 总是可以通过抢占运行在待考察 Node 上。</p>
<p>而我们需要执行第二遍 Predicates 算法的原因，则是因为“潜在的抢占者”最后不一定会运行在待考察的 Node 上。关于这一点，我在前面已经讲解过了：Kubernetes 调度器并不保证抢占者一定会运行在当初选定的被抢占的 Node 上。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color4">调度器</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/Kubernetes%E7%9F%A5%E8%AF%86%E6%8B%93%E5%B1%95-%E9%BB%98%E8%AE%A4%E8%B0%83%E5%BA%A6%E5%99%A8%E6%89%A7%E8%A1%8C%E9%80%BB%E8%BE%91/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s服务发布入门" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E6%9C%8D%E5%8A%A1%E5%8F%91%E5%B8%83%E5%85%A5%E9%97%A8/">k8s服务发布入门</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E6%9C%8D%E5%8A%A1%E5%8F%91%E5%B8%83%E5%85%A5%E9%97%A8/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="1-Label和-Selector"><a href="#1-Label和-Selector" class="headerlink" title="1  Label和 Selector"></a>1  Label和 Selector</h2><h3 id="1-1-什么是Label和Selector"><a href="#1-1-什么是Label和Selector" class="headerlink" title="1.1 什么是Label和Selector"></a>1.1 什么是Label和Selector</h3><p>当Kubernetes对系统的任何API对象如Pod和节点进行“分组”时，会对其添加Label（key=value形式的“键-值对”）用以精准地选择对应的API对象。</p>
<p>而Selector（标签选择器）则是针对匹配对象的查询方法。注：键-值对就是key-value pair。</p>
<p>例如，常用的标签：</p>
<ul>
<li>版本标签：”release” : “stable” 、”release” : “canary”</li>
<li>环境标签：”environment” : “dev” 、”environment” : “qa” 、 “environment” : “production”</li>
<li>架构标签：”tier” : “frontend”  、 “tier” : “backend” 、 “tier” : “middleware”</li>
<li>分区标签：”partition” : ‘customerA’ ,  “partition” : “customerB”</li>
<li>质量管控标签：”track” : “daily” 、”track” : “weekly”</li>
</ul>
<h3 id="1-2-Label的使用"><a href="#1-2-Label的使用" class="headerlink" title="1.2 Label的使用"></a>1.2 Label的使用</h3><h4 id="1-2-1-命令行定义Label"><a href="#1-2-1-命令行定义Label" class="headerlink" title="1.2.1 命令行定义Label"></a>1.2.1 命令行定义Label</h4><ul>
<li><p>格式</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label API对象 名称 key&#x3D;value</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>示例</p>
<p>公司与xx银行有一条专属的高速光纤通道，此通道只能与192.168.7.0网段进行通信，因此只能将与xx银行通信的应用部署到192.168.7.0网段所在的节点上，此时可以对节点进行Label（即加标签）：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label node k8s-node02 region&#x3D;subnet7</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="1-2-2-yaml定义Label"><a href="#1-2-2-yaml定义Label" class="headerlink" title="1.2.2 yaml定义Label"></a>1.2.2 yaml定义Label</h4><ul>
<li><p>以myweb Pod 为例， Label被定义在其metadata中：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">myweb</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">myweb</span></span><br><span class="line">    <span class="attr">release:</span> <span class="string">stable</span></span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="1-2-3-查看标签"><a href="#1-2-3-查看标签" class="headerlink" title="1.2.3 查看标签"></a>1.2.3 查看标签</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get svc --show-labels</span><br></pre></td></tr></table></figure>



<h4 id="1-2-4-修改标签"><a href="#1-2-4-修改标签" class="headerlink" title="1.2.4 修改标签"></a>1.2.4 修改标签</h4><p>对已有标签修改时，需要添加参数 <strong>–overwrite</strong>才能生效。</p>
<ul>
<li><p>在实际使用中，Label的更改是经常发生的事情，可以使用overwrite参数修改标签。</p>
<p>修改标签，比如将version=v1改为version=v2：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label svc canary-v1 -n canary-production version&#x3D;v2 --overwrite</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="1-2-5-删除标签"><a href="#1-2-5-删除标签" class="headerlink" title="1.2.5 删除标签"></a>1.2.5 删除标签</h4><p>在标签名的后面增加一个减号**-**，即为删除该标签</p>
<ul>
<li><p>删除version</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label svc canary-v1 -n canary-production version-</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="1-3-Selector条件匹配"><a href="#1-3-Selector条件匹配" class="headerlink" title="1.3 Selector条件匹配"></a>1.3 Selector条件匹配</h3><h4 id="1-3-1-Label-Selector"><a href="#1-3-1-Label-Selector" class="headerlink" title="1.3.1 Label Selector"></a>1.3.1 Label Selector</h4><p>Lable Selector 可以类比为sql语句中的where查询条件，例如，name=redis-slave，这个Label Selector 作用于Pod时，可以被类比为”select * from pod where name=redis-slave”</p>
<ul>
<li>下面是一些具体的例子：<ol>
<li>name = redis-slave：匹配所有具有标签name=redis-slave的例子</li>
<li>env != production: 匹配所有不具有标签env=production的资源对象，比如env=test就是满足此条件的标签之一。</li>
<li>name in ( redis-master, redis-slave) : 匹配所有具有标签name=redis-master或者name=redis-slave的资源对象</li>
<li>name notin (php-frontend): 匹配所有不具有标签name=php-frontend的资源对象。</li>
</ol>
</li>
</ul>
<ul>
<li><p>可以通过多个Label Selector表达式的组合实现复杂的条件选择，多个表达式之间用”,” 进行分割即可，几个条件直接是”AND”的关系，即同时满足多个条件</p>
<p>比如：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">name&#x3D;redis-slave,env!&#x3D;production</span><br><span class="line">name notin (php-frontend),env!&#x3D;production</span><br></pre></td></tr></table></figure>

</li>
</ul>
<blockquote>
<p>kubectl使用Label selector时，需要将表达放入””或’’中，如：kubectl get po -A -l ‘app notin (busybox)’</p>
</blockquote>
<h4 id="1-3-2-yaml中Selector的使用"><a href="#1-3-2-yaml中Selector的使用" class="headerlink" title="1.3.2 yaml中Selector的使用"></a>1.3.2 yaml中Selector的使用</h4><ul>
<li><p>资源对象service中管理Pod的Lable（详细用法见后续service）</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">service</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">myweb</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">myweb</span></span><br><span class="line">  <span class="attr">ports:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">port:</span> <span class="number">8080</span></span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>其他管理对象如Deployment、ReplicaSet、DaemonSet和Job则可以在Selector中使用基于集合的删选条件定义，例如：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">selector:</span><br><span class="line">  matchLabels:</span><br><span class="line">    app: myweb</span><br><span class="line">  matchExpressions:</span><br><span class="line">    - &#123;key: tier, operator: In, values: [frontend]&#125;</span><br><span class="line">    - &#123;key: environment, operator: NotIn, Values: [dev]&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li><p>matchLabels用于定义一组Label，与直接卸载Selector中的作用相同</p>
</li>
<li><p>matchExpressions用于定义一组基于集合的删选条件，可用的条件用算符包括In，NotIn，Exists 和 DoesNotExist</p>
</li>
<li><p>如果同时设置了matchLables 和 matchExpressions， 则两组条件为AND关系，即需要同时满足所有条件才能完成Selector的筛选</p>
</li>
</ul>
</li>
</ul>
<h2 id="2-Service"><a href="#2-Service" class="headerlink" title="2 Service"></a>2 Service</h2><h3 id="2-1-什么是service？"><a href="#2-1-什么是service？" class="headerlink" title="2.1 什么是service？"></a>2.1 什么是service？</h3><p>Kubernetes Pod具有生命周期的概念，它可以被创建、删除、销毁，一旦被销毁就意味着生命周期的结束。通过ReplicaSet能够动态地创建和销毁Pod，例如进行扩缩容和执行滚动升级。每个Pod都会获取到它自己的IP地址，但是这些IP地址不总是稳定和可依赖的，这样就会导致一个问题：在Kubernetes集群中，如果一组Pod（比如后端的Pod）为其他Pod（比如前端的Pod）提供服务，那么如果它们之间使用Pod的IP地址进行通信，在Pod重建后，将无法再进行连接。</p>
<p>为了解决上述问题，Kubernetes引用了Service这样一种抽象概念：逻辑上的一组Pod，即一种可以访问Pod的策略——通常称为微服务。这一组Pod能够被Service访问到，通常是<strong>通过Label Selector（标签选择器）实现</strong>的。</p>
<h3 id="2-2-什么是Endpoints？"><a href="#2-2-什么是Endpoints？" class="headerlink" title="2.2  什么是Endpoints？"></a>2.2  什么是Endpoints？</h3><p>在service创建时，自动创建的资源类型。记录了service匹配的pod的ip地址和端口，当pod重启删除等情况导致ip地址变更时就会自动更新。</p>
<ul>
<li><p>查询endpoints资源</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get ep -n default</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="2-3-定义Service"><a href="#2-3-定义Service" class="headerlink" title="2.3 定义Service"></a>2.3 定义Service</h3><ul>
<li><p>准备工作，创建一个nginx的deployment</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt;&gt; nginx-deployment.yaml</span><br><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  labels:</span><br><span class="line">    app: nginx</span><br><span class="line">  name: nginx</span><br><span class="line">  namespace: default</span><br><span class="line">spec:</span><br><span class="line">  progressDeadlineSeconds: 600</span><br><span class="line">  replicas: 2</span><br><span class="line">  revisionHistoryLimit: 10</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      app: nginx</span><br><span class="line">  strategy:</span><br><span class="line">    rollingUpdate:</span><br><span class="line">      maxSurge: 25%</span><br><span class="line">      maxUnavailable: 25%</span><br><span class="line">    type: RollingUpdate</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      creationTimestamp: null</span><br><span class="line">      labels:</span><br><span class="line">        app: nginx</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">      - image: nginx:1.15.2</span><br><span class="line">        imagePullPolicy: IfNotPresent</span><br><span class="line">        name: nginx</span><br><span class="line">        resources: &#123;&#125;</span><br><span class="line">        terminationMessagePath: &#x2F;dev&#x2F;termination-log</span><br><span class="line">        terminationMessagePolicy: File</span><br><span class="line">        terminationMessagePath: &#x2F;dev&#x2F;termination-log</span><br><span class="line">        terminationMessagePolicy: File</span><br><span class="line">      dnsPolicy: ClusterFirst</span><br><span class="line">      restartPolicy: Always</span><br><span class="line">      schedulerName: default-scheduler</span><br><span class="line">      securityContext: &#123;&#125;</span><br><span class="line">      terminationGracePeriodSeconds: 30</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line">kubectl create -f nginx-deployment.yaml</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="2-3-1-单端口service"><a href="#2-3-1-单端口service" class="headerlink" title="2.3.1 单端口service"></a>2.3.1 单端口service</h4><ul>
<li><p>定义service</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">kind: Service</span><br><span class="line">apiVersion: v1</span><br><span class="line">metadata:</span><br><span class="line">  name: nginx-service</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    app: nginx				# 匹配的pod标签</span><br><span class="line">  ports:</span><br><span class="line">    - protocol: TCP			# 网络协议TCP&#x2F;UDP等默认TCP</span><br><span class="line">      port: 80				# Service自己的端口</span><br><span class="line">      targetPort: 80		# 后端pod中应用的端口</span><br></pre></td></tr></table></figure>

</li>
</ul>
<p>上述配置创建一个名为nginx-service的Service对象，它会将请求代理到TCP端口为9376并且具有标签app=nginx的Pod上。这个Service会被分配一个IP地址，通常称为ClusterIP，它会被服务的代理使用。</p>
<blockquote>
<p>需要注意的是，Service能够将一个接收端口映射到任意的targetPort。默认情况下，targetPort将被设置为与Port字段相同的值。targetPort可以设置为一个字符串，引用backend Pod的一个端口的名称。</p>
</blockquote>
<h4 id="2-3-2-没有selector的-Service（代理k8s外部应用）"><a href="#2-3-2-没有selector的-Service（代理k8s外部应用）" class="headerlink" title="2.3.2 没有selector的 Service（代理k8s外部应用）"></a>2.3.2 没有selector的 Service（代理k8s外部应用）</h4><p>服务最常见的是抽象化对 Kubernetes Pod 的访问，但是它们也可以抽象化其他种类的后端。 实例:</p>
<ul>
<li>希望在生产环境中使用外部的数据库集群，但测试环境使用自己的数据库。</li>
<li>希望服务指向另一个命名空间中或其它集群中的服务。</li>
<li>你正在将工作负载迁移到 Kubernetes。 在评估该方法时，你仅在 Kubernetes 中运行一部分后端。</li>
</ul>
<p>在任何这些场景中，都能够定义没有selector的 Service。 实例:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: nginx-service</span><br><span class="line">spec:</span><br><span class="line">  ports:</span><br><span class="line">    - protocol: TCP</span><br><span class="line">      port: 80</span><br><span class="line">      targetPort: 80</span><br></pre></td></tr></table></figure>



<p>由于此服务没有selector，因此不会自动创建相应的 Endpoint 对象。 你可以通过手动添加 Endpoint 对象，将服务手动映射到运行该服务的网络地址和端口：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Endpoints</span><br><span class="line">metadata:</span><br><span class="line">  name: my-service</span><br><span class="line">subsets:</span><br><span class="line">  - addresses:</span><br><span class="line">      - ip: 10.244.85.208</span><br><span class="line">    ports:</span><br><span class="line">      - port: 80</span><br></pre></td></tr></table></figure>

<p>访问没有选择算符的 Service，与有选择算符的 Service 的原理相同。 请求将被路由到用户定义的 Endpoint，YAML 中为：<code>10.244.85.208:80</code>（TCP）。</p>
<h4 id="2-3-3-多端口的service"><a href="#2-3-3-多端口的service" class="headerlink" title="2.3.3 多端口的service"></a>2.3.3 多端口的service</h4><p>对于某些服务，你需要公开多个端口。 Kubernetes 允许你在 Service 对象上配置多个端口定义。 为服务使用多个端口时，必须提供所有端口名称，以使它们无歧义。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: nginx-service</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    app: nginx</span><br><span class="line">  ports:</span><br><span class="line">    - name: http</span><br><span class="line">      protocol: TCP</span><br><span class="line">      port: 80</span><br><span class="line">      targetPort: 80</span><br><span class="line">    - name: https</span><br><span class="line">      protocol: TCP</span><br><span class="line">      port: 443</span><br><span class="line">      targetPort: 443</span><br></pre></td></tr></table></figure>



<blockquote>
<p>说明：</p>
<p>​        与一般的Kubernetes名称一样，端口名称只能包含小写字母数字字符 和 <code>-</code>。 端口名称还必须以字母数字字符开头和结尾。</p>
<p>​        例如，名称 <code>123-abc</code> 和 <code>web</code> 有效，但是 <code>123_abc</code> 和 <code>-web</code> 无效。</p>
</blockquote>
<h4 id="2-3-4-使用Service反代域名"><a href="#2-3-4-使用Service反代域名" class="headerlink" title="2.3.4 使用Service反代域名"></a>2.3.4 使用Service反代域名</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  labels:</span><br><span class="line">    app: nginx-externalname</span><br><span class="line">  name: nginx-externalname</span><br><span class="line">spec:</span><br><span class="line">  type: ExternalName</span><br><span class="line">  externalName: www.baidu.com</span><br></pre></td></tr></table></figure>







<h3 id="2-4-访问service"><a href="#2-4-访问service" class="headerlink" title="2.4 访问service"></a>2.4 访问service</h3><h4 id="2-4-1-使用clusterIp访问service"><a href="#2-4-1-使用clusterIp访问service" class="headerlink" title="2.4.1 使用clusterIp访问service"></a>2.4.1 使用clusterIp访问service</h4><p>通过<code>kubectl get svc -n namespace servicename</code>查看clusterip就可以直接通过clusterip访问service代理的后端pod</p>
<ul>
<li>查看cluster-ip</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ kubectl get svc -n default nginx-service</span><br><span class="line">NAME            TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE</span><br><span class="line">kubernetes      ClusterIP   10.96.0.1      &lt;none&gt;        443&#x2F;TCP   57d</span><br><span class="line">nginx-service   ClusterIP   10.109.35.79   &lt;none&gt;        80&#x2F;TCP    44s</span><br></pre></td></tr></table></figure>

<ul>
<li>访问</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl 10.109.35.79</span><br></pre></td></tr></table></figure>





<h4 id="2-4-2-同namespace下在pod中通过service的名称访问"><a href="#2-4-2-同namespace下在pod中通过service的名称访问" class="headerlink" title="2.4.2 同namespace下在pod中通过service的名称访问"></a>2.4.2 同namespace下在pod中通过service的名称访问</h4><ul>
<li><p>如下service名称为nginx-serivce，进入同namespace的pod中访问service</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">wget http:&#x2F;&#x2F;nginx-service</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="2-4-3-在pod中跨namespace访问"><a href="#2-4-3-在pod中跨namespace访问" class="headerlink" title="2.4.3 在pod中跨namespace访问"></a>2.4.3 在pod中跨namespace访问</h4><ul>
<li><p>在service名称后紧跟<code>.namespace</code>就可以跨namespace访问，如下访问default的namespace中的service：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">wget http:&#x2F;&#x2F;nginx-service.default</span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="2-5-service服务类型"><a href="#2-5-service服务类型" class="headerlink" title="2.5 service服务类型"></a>2.5 service服务类型</h3><p>对一些应用的某些部分（如前端），可能希望将其暴露给 Kubernetes 集群外部 的 IP 地址。</p>
<p>Kubernetes <code>ServiceTypes</code> 允许指定你所需要的 Service 类型，默认是 <code>ClusterIP</code>。</p>
<p><code>Type</code> 的取值以及行为如下：</p>
<ul>
<li><code>ClusterIP</code>：通过集群的内部 IP 暴露服务，选择该值时服务只能够在集群内部访问。 这也是默认的 <code>ServiceType</code>。</li>
<li><code>NodePort</code>：通过每个节点上的 IP 和静态端口（<code>NodePort</code>）暴露服务。 <code>NodePort</code> 服务会路由到自动创建的 <code>ClusterIP</code> 服务。 通过请求 <code>&lt;节点 IP&gt;:&lt;节点端口&gt;</code>，你可以从集群的外部访问一个 <code>NodePort</code> 服务。</li>
<li><code>LoadBalancer</code>：使用云提供商的负载均衡器向外部暴露服务。 外部负载均衡器可以将流量路由到自动创建的 <code>NodePort</code> 服务和 <code>ClusterIP</code> 服务上。</li>
<li><code>ExternalName</code>：通过返回 <code>CNAME</code> 和对应值，可以将服务映射到 <code>externalName</code> 字段的内容（例如，<code>foo.bar.example.com</code>）。 无需创建任何类型代理。</li>
</ul>
<h4 id="2-5-1-NodePort-类型"><a href="#2-5-1-NodePort-类型" class="headerlink" title="2.5.1 NodePort 类型"></a>2.5.1 NodePort 类型</h4><p>如果你将 <code>type</code> 字段设置为 <code>NodePort</code>，则 Kubernetes 控制平面将在 <code>--service-node-port-range</code> 标志指定的范围内分配端口（默认值：30000-32767）。 每个节点将那个端口（每个节点上的相同端口号）代理到你的服务中。 你的服务在其 <code>.spec.ports[*].nodePort</code> 字段中要求分配的端口。</p>
<p>如果需要特定的端口号，你可以在 <code>nodePort</code> 字段中指定一个值。 控制平面将为你分配该端口或报告 API 事务失败。 这意味着你需要自己注意可能发生的端口冲突。 你还必须使用有效的端口号，该端口号在配置用于 NodePort 的范围内。</p>
<p>需要注意的是，Service 能够通过 <code>&lt;NodeIP&gt;:spec.ports[*].nodePort</code> 和 <code>spec.clusterIp:spec.ports[*].port</code> 而对外可见。</p>
<ul>
<li>例如：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: my-service</span><br><span class="line">spec:</span><br><span class="line">  type: NodePort</span><br><span class="line">  selector:</span><br><span class="line">    app: MyApp</span><br><span class="line">  ports:</span><br><span class="line">      # 默认情况下，为了方便起见，&#96;targetPort&#96; 被设置为与 &#96;port&#96; 字段相同的值。</span><br><span class="line">    - port: 80</span><br><span class="line">      targetPort: 80</span><br><span class="line">      # 可选字段</span><br><span class="line">      # 默认情况下，为了方便起见，Kubernetes 控制平面会从某个范围内分配一个端口号（默认：30000-32767）</span><br><span class="line">      nodePort: 30007</span><br></pre></td></tr></table></figure>



<h4 id="2-5-2-ClusterIP-类型"><a href="#2-5-2-ClusterIP-类型" class="headerlink" title="2.5.2 ClusterIP 类型"></a>2.5.2 ClusterIP 类型</h4><p>详见2.3.1</p>
<h4 id="2-5-3-ExternalName"><a href="#2-5-3-ExternalName" class="headerlink" title="2.5.3 ExternalName"></a>2.5.3 ExternalName</h4><p>类型为 ExternalName 的服务将服务映射到 DNS 名称，而不是典型的选择器，例如 <code>my-service</code> 或者 <code>cassandra</code>。 你可以使用 <code>spec.externalName</code> 参数指定这些服务。</p>
<p>详见2.3.4</p>
<h4 id="2-5-4-绑定公网IP"><a href="#2-5-4-绑定公网IP" class="headerlink" title="2.5.4 绑定公网IP"></a>2.5.4 绑定公网IP</h4><p>如果有一些公网IP 地址能够访问到至少一个集群节点，Kubernetes 服务就可以在这些 <code>externalIPs</code> 上暴露出来。 </p>
<p>通过外部 IP 进入集群的入站请求，如果指向的是服务的端口，会被路由到服务的后端之一。 </p>
<p><code>externalIPs</code> 不受 Kubernets 管理；它们由集群管理员管理。 在服务规约中，<code>externalIPs</code> 可以和 <code>ServiceTypes</code> 一起指定。 在下面的例子中，客户端可以通过 “<code>80.11.12.10:80</code>“ (<code>externalIP:port</code>) 访问 “<code>my-service</code>“ 服务。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">kind: Service</span><br><span class="line">apiVersion: v1</span><br><span class="line">metadata:</span><br><span class="line">  name: my-service</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    app: MyApp</span><br><span class="line">  ports:</span><br><span class="line">  - name: http</span><br><span class="line">    protocol: TCP</span><br><span class="line">    port: 80</span><br><span class="line">    targetPort: 9376</span><br><span class="line">  externalIPs:</span><br><span class="line">  - 80.11.12.10</span><br></pre></td></tr></table></figure>





<h4 id="2-5-5-无头服务（Headless-Services）"><a href="#2-5-5-无头服务（Headless-Services）" class="headerlink" title="2.5.5 无头服务（Headless Services）"></a>2.5.5 无头服务（Headless Services）</h4><p>有时不需要或不想要负载均衡，以及单独的 Service IP。 遇到这种情况，可以通过指定 Cluster IP（<code>spec.clusterIP</code>）的值为 <code>&quot;None&quot;</code> 来创建 <code>Headless</code> Service。</p>
<p>你可以使用无头 Service 与其他服务发现机制进行接口，而不必与 Kubernetes 的实现捆绑在一起。</p>
<p>对这无头 Service 并不会分配 Cluster IP，kube-proxy 不会处理它们， 而且平台也不会为它们进行负载均衡和路由。 DNS 如何实现自动配置，依赖于 Service 是否定义了Selector。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:  </span><br><span class="line">  name: nginx-prod-demo-headless-svc</span><br><span class="line">  namespace: demo </span><br><span class="line">spec:</span><br><span class="line">  clusterIP: None	# 无头服务（Headless Services）</span><br><span class="line">  selector: </span><br><span class="line">    app: nginx</span><br><span class="line">    env: prod</span><br><span class="line">  type: ClusterIP</span><br><span class="line">  ports:  </span><br><span class="line">  - name: http </span><br><span class="line">    port: 80 </span><br><span class="line">    targetPort: 80 </span><br><span class="line">    protocol: TCP </span><br></pre></td></tr></table></figure>

<p>对于指定了 clusterIP=None 的 Headless Service 来说，它的 A 记录的格式也是：..svc.cluster.local。但是，当你访问这条 A 记录的时候，它返回的是所有被代理的 Pod 的 IP 地址的集合。当然，如果你的客户端没办法解析这个集合的话，它可能会只会拿到第一个 Pod 的 IP 地址。</p>
<p>如果你为 Pod 指定了 Headless Service，并且 Pod 本身声明了 hostname 和 subdomain 字段，那么这时候 Pod 的 A 记录就会变成：…svc.cluster.local，比如：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: default-subdomain</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    name: busybox</span><br><span class="line">  clusterIP: None</span><br><span class="line">  ports:</span><br><span class="line">  - name: foo</span><br><span class="line">    port: 1234</span><br><span class="line">    targetPort: 1234</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: busybox1</span><br><span class="line">  labels:</span><br><span class="line">    name: busybox</span><br><span class="line">spec:</span><br><span class="line">  hostname: busybox-1</span><br><span class="line">  subdomain: default-subdomain</span><br><span class="line">  containers:</span><br><span class="line">  - image: busybox</span><br><span class="line">    command:</span><br><span class="line">      - sleep</span><br><span class="line">      - &quot;3600&quot;</span><br><span class="line">    name: busybox</span><br></pre></td></tr></table></figure>

<p>在上面这个 Service 和 Pod 被创建之后，你就可以通过 busybox-1.default-subdomain.default.svc.cluster.local 解析到这个 Pod 的 IP 地址了。</p>
<blockquote>
<p>关于 hostname 和 subdomain 字段祥见后续2.6</p>
</blockquote>
<p>总结下， Headless Service 主要有如下两种场景:</p>
<ol>
<li>用户可以自己选择要连接哪个 Pod，通过查询 Service 的 DNS 记录来获取后端真实负载的 IP 地址，自主选择要连接哪个 IP；</li>
<li>可用于部署有状态服务。在 StatefulSet 中也有 Headless Service 例子，每个 StatefulSet 管理的 Pod 都有一个单独的 DNS 记录，且域名保持不变，即<PodName>.<ServiceName>.<NamespaceName>.svc.cluster.local。这样 Statefulset 中的各个 Pod 就可以直接通过 Pod 名字解决相互间身份以及访问问题。</li>
</ol>
<p><strong>clusterIP 模式的 Service 为你提供的，就是一个 Pod 的稳定的 IP 地址，即 VIP。并且，这里 Pod 和 Service 的关系是可以通过 Label 确定的。</strong></p>
<p><strong>而 Headless Service 为你提供的，则是一个 Pod 的稳定的 DNS 名字，并且，这个名字是可以通过 Pod 名字和 Service 名字拼接出来的。</strong></p>
<h3 id="2-6-Pod-的-hostname-和-subdomain-字段"><a href="#2-6-Pod-的-hostname-和-subdomain-字段" class="headerlink" title="2.6 Pod 的 hostname 和 subdomain 字段"></a>2.6 Pod 的 hostname 和 subdomain 字段</h3><p>默认情况下，创建 Pod 时其主机名取自 Pod 的 <code>metadata.name</code> 值。</p>
<p>Pod 规约中包含一个可选的 <code>hostname</code> 字段，可以用来指定 Pod 的主机名。 当这个字段被设置时，它将优先于 Pod 的名字成为该 Pod 的主机名。 举个例子，给定一个 <code>hostname</code> 设置为 “<code>my-host</code>“ 的 Pod， 该 Pod 的主机名将被设置为 “<code>my-host</code>“。</p>
<p>Pod 规约还有一个可选的 <code>subdomain</code> 字段，可以用来指定 Pod 的子域名。 举个例子，某 Pod 的 <code>hostname</code> 设置为 “<code>foo</code>”，<code>subdomain</code> 设置为 “<code>bar</code>”， 在名字空间 “<code>my-namespace</code>” 中对应的完全限定域名（FQDN）为 “<code>foo.bar.my-namespace.svc.cluster.local</code>”。</p>
<p>示例：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: default-subdomain</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    name: busybox</span><br><span class="line">  clusterIP: None</span><br><span class="line">  ports:</span><br><span class="line">  - name: foo # 实际上不需要指定端口号</span><br><span class="line">    port: 1234</span><br><span class="line">    targetPort: 1234</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: busybox1</span><br><span class="line">  labels:</span><br><span class="line">    name: busybox</span><br><span class="line">spec:</span><br><span class="line">  hostname: busybox-1</span><br><span class="line">  subdomain: default-subdomain</span><br><span class="line">  containers:</span><br><span class="line">  - image: busybox:1.28</span><br><span class="line">    command:</span><br><span class="line">      - sleep</span><br><span class="line">      - &quot;3600&quot;</span><br><span class="line">    name: busybox</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: busybox2</span><br><span class="line">  labels:</span><br><span class="line">    name: busybox</span><br><span class="line">spec:</span><br><span class="line">  hostname: busybox-2</span><br><span class="line">  subdomain: default-subdomain</span><br><span class="line">  containers:</span><br><span class="line">  - image: busybox:1.28</span><br><span class="line">    command:</span><br><span class="line">      - sleep</span><br><span class="line">      - &quot;3600&quot;</span><br><span class="line">    name: busybox</span><br></pre></td></tr></table></figure>

<p>如果某无头服务与某 Pod 在同一个名字空间中，且它们具有相同的子域名， 集群的 DNS 服务器也会为该 Pod 的全限定主机名返回 A 记录或 AAAA 记录。 例如，在同一个名字空间中，给定一个主机名为 “busybox-1”、 子域名设置为 “default-subdomain” 的 Pod，和一个名称为 “<code>default-subdomain</code>” 的无头服务，Pod 将看到自己的 FQDN 为 “<code>busybox-1.default-subdomain.my-namespace.svc.cluster.local</code>“。 DNS 会为此名字提供一个 A 记录或 AAAA 记录，指向该 Pod 的 IP。 “<code>busybox1</code>” 和 “<code>busybox2</code>” 这两个 Pod 分别具有它们自己的 A 或 AAAA 记录。</p>
<h3 id="2-7-关于service负载均衡算法"><a href="#2-7-关于service负载均衡算法" class="headerlink" title="2.7 关于service负载均衡算法"></a>2.7 关于service负载均衡算法</h3><p>kubernets提供了两种负载均衡算法：</p>
<ul>
<li>RoundRobin：轮询模式，即轮询将请求转发到后端的pod上。</li>
<li>SessionAffinity：基于客户端IP地址进行会话保持的模式</li>
</ul>
<p>在默认情况下，kubernets采用RoundRobin模式对客户端请求进行负载分发，带我们也可以通过设置<code>service.spec.sessionAffinity=ClientIP</code>来弃用SessionAffinity策略。</p>
<h2 id="3-Ingress"><a href="#3-Ingress" class="headerlink" title="3 Ingress"></a>3 Ingress</h2><h3 id="3-1-什么是Ingress？"><a href="#3-1-什么是Ingress？" class="headerlink" title="3.1 什么是Ingress？"></a>3.1 什么是Ingress？</h3><p>Ingress为Kubernetes集群中的服务提供了入口，可以提供负载均衡、SSL终止和基于名称的虚拟主机，在生产环境中常用的Ingress有Treafik、Nginx、HAProxy、Istio等。</p>
<p>在Kubernetesv 1.1版中添加的Ingress用于从集群外部到集群内部Service的HTTP和HTTPS路由，流量从Internet到Ingress再到Services最后到Pod上，通常情况下，Ingress部署在所有的Node节点上。</p>
<p>Ingress可以配置提供服务外部访问的URL、负载均衡、终止SSL，并提供基于域名的虚拟主机。但Ingress不会暴露任意端口或协议。</p>
<h3 id="3-2-helm安装ingress"><a href="#3-2-helm安装ingress" class="headerlink" title="3.2 helm安装ingress"></a>3.2 helm安装ingress</h3><p>首先安装helm管理工具参考：<a target="_blank" rel="noopener" href="https://helm.sh/docs/intro/install/">https://helm.sh/docs/intro/install/</a></p>
<p>使用helm安装ingress参考：<a target="_blank" rel="noopener" href="https://kubernetes.github.io/ingress-nginx/deploy/#using-helm">https://kubernetes.github.io/ingress-nginx/deploy/#using-helm</a></p>
<h4 id="3-2-1-添加ingress的helm仓库"><a href="#3-2-1-添加ingress的helm仓库" class="headerlink" title="3.2.1 添加ingress的helm仓库"></a>3.2.1 添加ingress的helm仓库</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">helm repo add ingress-nginx https:&#x2F;&#x2F;kubernetes.github.io&#x2F;ingress-nginx</span><br></pre></td></tr></table></figure>



<h4 id="3-2-2-下载ingress的helm包至本地"><a href="#3-2-2-下载ingress的helm包至本地" class="headerlink" title="3.2.2 下载ingress的helm包至本地"></a>3.2.2 下载ingress的helm包至本地</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">helm pull ingress-nginx&#x2F;ingress-nginx</span><br></pre></td></tr></table></figure>



<h4 id="3-2-3-更改对应的配置"><a href="#3-2-3-更改对应的配置" class="headerlink" title="3.2.3  更改对应的配置"></a>3.2.3  更改对应的配置</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">tar xf ingress-nginx-3.20.1.tgz</span><br><span class="line"></span><br><span class="line">cd ingress-nginx</span><br><span class="line"></span><br><span class="line">vim values.yaml</span><br></pre></td></tr></table></figure>

<p>需要修改的位置如下：</p>
<ol>
<li>Controller和admissionWebhook的镜像地址，需要将公网镜像同步至公司内网镜像仓库（和课程不一致的版本，需要自行同步gcr镜像的，可以百度查一下使用阿里云同步gcr的镜像，也可以参考这个连接<a target="_blank" rel="noopener" href="https://blog.csdn.net/weixin_39961559/article/details/80739352%EF%BC%89">https://blog.csdn.net/weixin_39961559/article/details/80739352）</a></li>
<li>k8s使用私有仓库时，需要手动创建secret，并在imagePullSecrets处指定secret，否则需要手动拉取镜像。</li>
<li>hostNetwork设置为true</li>
<li>dnsPolicy设置为 ClusterFirstWithHostNet</li>
<li>nodeSelector添加ingress: “true”部署至指定节点</li>
<li>类型更改为kind: DaemonSet</li>
<li>type: 设置为ClusterIP</li>
</ol>
<h4 id="3-2-4-部署ingress-controller"><a href="#3-2-4-部署ingress-controller" class="headerlink" title="3.2.4 部署ingress controller"></a>3.2.4 部署ingress controller</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">#需要部署ingress的node添加标签</span><br><span class="line">kubectl label node k8s-node01 ingress&#x3D;true</span><br><span class="line"></span><br><span class="line">#创建ingress的namespace</span><br><span class="line">kubectl create ns ingress-nginx</span><br><span class="line"></span><br><span class="line">#安装ingress controller</span><br><span class="line">helm install ingress-nginx -n ingress-nginx .</span><br></pre></td></tr></table></figure>





<h3 id="3-3-Ingress入门使用"><a href="#3-3-Ingress入门使用" class="headerlink" title="3.3 Ingress入门使用"></a>3.3 Ingress入门使用</h3><h4 id="3-3-1-创建一个简单的ingress"><a href="#3-3-1-创建一个简单的ingress" class="headerlink" title="3.3.1 创建一个简单的ingress"></a>3.3.1 创建一个简单的ingress</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: networking.k8s.io&#x2F;v1beta1 	# 新版本推荐 networking.k8s.io&#x2F;v1    已被弃用extensions&#x2F;v1beta1</span><br><span class="line">kind: Ingress</span><br><span class="line">metadata:</span><br><span class="line">  annotations: </span><br><span class="line">    kubernetes.io&#x2F;ingress.class: &quot;nginx&quot;</span><br><span class="line">  name: example</span><br><span class="line">spec:</span><br><span class="line">  rules: 							    # 一个Ingress可以配置多个rules</span><br><span class="line">  - host: foo.bar.com 					# 域名配置，可以不写，匹配*， *.bar.com</span><br><span class="line">    http:</span><br><span class="line">      paths: 							# 相当于nginx的location配合，同一个host可以配置多个path &#x2F; &#x2F;abc</span><br><span class="line">      - backend:</span><br><span class="line">          serviceName: nginx-svc </span><br><span class="line">          servicePort: 80</span><br><span class="line">        path: &#x2F;</span><br></pre></td></tr></table></figure>





<h4 id="3-3-2-多域名ingress"><a href="#3-3-2-多域名ingress" class="headerlink" title="3.3.2 多域名ingress"></a>3.3.2 多域名ingress</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: networking.k8s.io&#x2F;v1beta1 # networking.k8s.io&#x2F;v1 &#x2F; extensions&#x2F;v1beta1 </span><br><span class="line">kind: Ingress</span><br><span class="line">metadata:</span><br><span class="line">  annotations:</span><br><span class="line">    kubernetes.io&#x2F;ingress.class: &quot;nginx&quot;</span><br><span class="line">  name: example</span><br><span class="line">spec:</span><br><span class="line">  rules: # 一个Ingress可以配置多个rules</span><br><span class="line">  - host: foo.bar.com # 域名配置，可以不写，匹配*， *.bar.com</span><br><span class="line">    http:</span><br><span class="line">      paths: # 相当于nginx的location配合，同一个host可以配置多个path &#x2F; &#x2F;abc</span><br><span class="line">      - backend:</span><br><span class="line">          serviceName: nginx-svc </span><br><span class="line">          servicePort: 80</span><br><span class="line">        path: &#x2F;</span><br><span class="line">  - host: foo2.bar.com # 域名配置，可以不写，匹配*， *.bar.com</span><br><span class="line">    http:</span><br><span class="line">      paths: # 相当于nginx的location配合，同一个host可以配置多个path &#x2F; &#x2F;abc</span><br><span class="line">      - backend:</span><br><span class="line">          serviceName: nginx-svc-external</span><br><span class="line">          servicePort: 80</span><br><span class="line">        path: &#x2F;</span><br></pre></td></tr></table></figure>



<h4 id="3-3-3-基于TLS的Ingress"><a href="#3-3-3-基于TLS的Ingress" class="headerlink" title="3.3.3 基于TLS的Ingress"></a>3.3.3 基于TLS的Ingress</h4><ul>
<li>首先创建证书，生产环境的证书为公司购买的证书：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl -n default create secret tls nginx-test-tls --key&#x3D;tls.key --cert&#x3D;tls.crt</span><br></pre></td></tr></table></figure>





<ul>
<li>定义Ingress（此示例为Traefik，nginx-ingress将traefik改为nginx即可）：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: networking.k8s.io&#x2F;v1beta1</span><br><span class="line">kind: Ingress</span><br><span class="line">metadata:</span><br><span class="line">  name: nginx-https-test</span><br><span class="line">  namespace: default</span><br><span class="line">  annotations:</span><br><span class="line">    kubernetes.io&#x2F;ingress.class: traefik</span><br><span class="line">spec:</span><br><span class="line">  rules:</span><br><span class="line">  - host: traefix-test.com</span><br><span class="line">    http:</span><br><span class="line">      paths:</span><br><span class="line">      - backend:</span><br><span class="line">          serviceName: nginx-svc</span><br><span class="line">          servicePort: 80</span><br><span class="line">  tls:</span><br><span class="line">   - secretName: nginx-test-tls</span><br></pre></td></tr></table></figure>


      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">service</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E6%9C%8D%E5%8A%A1%E5%8F%91%E5%B8%83%E5%85%A5%E9%97%A8/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s基础持久化存储Volumes&amp;pv&amp;pvc" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8Volumes&pv&pvc/">k8s基础持久化存储Volumesd&amp;pv&amp;pvc</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8Volumes&pv&pvc/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-volumes"><a href="#1-volumes" class="headerlink" title="1 volumes"></a>1 volumes</h1><p>Container 中的文件在磁盘上是临时存放的，这给 Container 中运行的较重要的应用程序带来一些问题。问题之一是当容器崩溃时文件丢失。kubelet 会重新启动容器， 但容器会以干净的状态重启。 第二个问题会在同一 <code>Pod</code> 中运行多个容器并共享文件时出现。 Kubernetes <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/volumes/">卷（Volume）</a> 这一抽象概念能够解决这两个问题。</p>
<h2 id="1-1-卷类型"><a href="#1-1-卷类型" class="headerlink" title="1.1 卷类型"></a>1.1 卷类型</h2><h3 id="1-1-1-emptyDir"><a href="#1-1-1-emptyDir" class="headerlink" title="1.1.1 emptyDir"></a>1.1.1 emptyDir</h3><h4 id="1-1-1-1-emptyDir介绍"><a href="#1-1-1-1-emptyDir介绍" class="headerlink" title="1.1.1.1 emptyDir介绍"></a>1.1.1.1 emptyDir介绍</h4><p>当 Pod 分配到某个 Node 上时，<code>emptyDir</code> 卷会被创建，并且Pod 在该节点上运行期间，该卷会一直存在。 另外就行它的名称表示的那样，<code>emptyDir</code>卷最初是空的。 </p>
<p>尽管 Pod 中的容器挂载 <code>emptyDir</code> 卷的路径可能相同也可能不同，这些容器都可以读写 <code>emptyDir</code> 卷中相同的文件。 当 Pod 因为某些原因被从节点上删除时，<code>emptyDir</code> 卷中的数据也会被永久删除。</p>
<blockquote>
<p>容器崩溃并<strong>不</strong>会导致 Pod 被从节点上移除，因此容器崩溃期间 <code>emptyDir</code> 卷中的数据是安全的。</p>
</blockquote>
<p><code>emptyDir</code> 的一些用途：</p>
<ul>
<li>缓存空间，例如基于磁盘的归并排序。</li>
<li>为耗时较长的计算任务提供检查点，以便任务能方便地从崩溃前状态恢复执行。</li>
<li>在 Web 服务器容器服务数据时，保存内容管理器容器获取的文件。</li>
</ul>
<p><code>emptyDir</code> 卷存储在该节点所使用的介质,取决于你的环境。可以是磁盘或 SSD 或网络存储。但是，你可以将 <code>emptyDir.medium</code> 字段设置为 <code>&quot;Memory&quot;</code>，以告诉 Kubernetes 为你挂载 tmpfs（基于 RAM 的文件系统）。 虽然 tmpfs 速度非常快，但是要注意它与磁盘不同。 tmpfs 在节点重启时会被清除，并且你所写入的所有文件都会计入容器的内存消耗，受容器内存限制约束。</p>
<h4 id="1-1-1-2-emptyDir-配置示例"><a href="#1-1-1-2-emptyDir-配置示例" class="headerlink" title="1.1.1.2 emptyDir 配置示例"></a>1.1.1.2 emptyDir 配置示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">test-pd</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">images:</span> <span class="string">k8s.gcr.io/test-webserver</span></span><br><span class="line">    <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/cache</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">cache-volume</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">cache-volume</span></span><br><span class="line">   <span class="attr">emptyDir:</span> &#123;&#125;</span><br></pre></td></tr></table></figure>



<h3 id="1-1-2-hostPath"><a href="#1-1-2-hostPath" class="headerlink" title="1.1.2 hostPath"></a>1.1.2 hostPath</h3><h4 id="1-1-2-1-hostPath介绍"><a href="#1-1-2-1-hostPath介绍" class="headerlink" title="1.1.2.1 hostPath介绍"></a>1.1.2.1 hostPath介绍</h4><p><code>hostPath</code> 卷能将主机节点文件系统上的文件或目录挂载到你的 Pod 中。 虽然这不是大多数 Pod 需要的，但是它为一些应用程序提供了强大的逃生舱。</p>
<p>例如，<code>hostPath</code> 的一些用法有：</p>
<ul>
<li>运行一个需要访问 Docker 内部机制的容器；可使用 <code>hostPath</code> 挂载 <code>/var/lib/docker</code> 路径。</li>
<li>在容器中运行 cAdvisor 时，以 <code>hostPath</code> 方式挂载 <code>/sys</code>。</li>
<li>允许 Pod 指定给定的 <code>hostPath</code> 在运行 Pod 之前是否应该存在，是否应该创建以及应该以什么方式存在。</li>
</ul>
<p>除了必需的 <code>path</code> 属性之外，用户可以选择性地为 <code>hostPath</code> 卷指定 <code>type</code>。</p>
<p>支持的 <code>type</code> 值如下：</p>
<table>
<thead>
<tr>
<th align="left">取值</th>
<th align="left">行为</th>
</tr>
</thead>
<tbody><tr>
<td align="left"></td>
<td align="left">空字符串（默认）用于向后兼容，这意味着在安装 hostPath 卷之前不会执行任何检查。</td>
</tr>
<tr>
<td align="left"><code>DirectoryOrCreate</code></td>
<td align="left">如果在给定路径上什么都不存在，那么将根据需要创建空目录，权限设置为 0755，具有与 kubelet 相同的组和属主信息。</td>
</tr>
<tr>
<td align="left"><code>Directory</code></td>
<td align="left">在给定路径上必须存在的目录。</td>
</tr>
<tr>
<td align="left"><code>FileOrCreate</code></td>
<td align="left">如果在给定路径上什么都不存在，那么将在那里根据需要创建空文件，权限设置为 0644，具有与 kubelet 相同的组和所有权。</td>
</tr>
<tr>
<td align="left"><code>File</code></td>
<td align="left">在给定路径上必须存在的文件。</td>
</tr>
<tr>
<td align="left"><code>Socket</code></td>
<td align="left">在给定路径上必须存在的 UNIX 套接字。</td>
</tr>
<tr>
<td align="left"><code>CharDevice</code></td>
<td align="left">在给定路径上必须存在的字符设备。</td>
</tr>
<tr>
<td align="left"><code>BlockDevice</code></td>
<td align="left">在给定路径上必须存在的块设备。</td>
</tr>
</tbody></table>
<h4 id="1-1-2-2-注意事项"><a href="#1-1-2-2-注意事项" class="headerlink" title="1.1.2.2 注意事项"></a>1.1.2.2 注意事项</h4><p>当使用这种类型的卷时要小心，因为：</p>
<ul>
<li>具有相同配置（例如基于同一 PodTemplate 创建）的多个 Pod 会由于节点上文件的不同 而在不同节点上有不同的行为。</li>
<li>下层主机上创建的文件或目录只能由 root 用户写入。你需要在 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/security-context/">特权容器</a> 中以 root 身份运行进程，或者修改主机上的文件权限以便容器能够写入 <code>hostPath</code> 卷。</li>
</ul>
<h4 id="1-1-2-2-hostPath-Directory配置示例"><a href="#1-1-2-2-hostPath-Directory配置示例" class="headerlink" title="1.1.2.2 hostPath Directory配置示例"></a>1.1.2.2 hostPath Directory配置示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">test-pd</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">k8s.gcr.io/test-webserver</span></span><br><span class="line">    <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/test-pd</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">test-volume</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-volume</span></span><br><span class="line">    <span class="attr">hostPath:</span></span><br><span class="line">      <span class="comment"># 宿主上目录位置</span></span><br><span class="line">      <span class="attr">path:</span> <span class="string">/data</span></span><br><span class="line">      <span class="comment"># 此字段为可选</span></span><br><span class="line">      <span class="attr">type:</span> <span class="string">Directory</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>注意：</strong> <code>FileOrCreate</code> 模式不会负责创建文件的父目录。 如果欲挂载的文件的父目录不存在，Pod 启动会失败。 为了确保这种模式能够工作，可以尝试把文件和它对应的目录分开挂载，如 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/volumes/#hostpath-fileorcreate-example"><code>FileOrCreate</code> 配置</a> 所示。</p>
</blockquote>
<h4 id="1-1-2-3-hostPath-FileOrCreate-配置示例"><a href="#1-1-2-3-hostPath-FileOrCreate-配置示例" class="headerlink" title="1.1.2.3 hostPath FileOrCreate 配置示例"></a>1.1.2.3 hostPath FileOrCreate 配置示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">test-webserver</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-webserver</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">k8s.gcr.io/test-webserver:latest</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/var/local/aaa</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">mydir</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/var/local/aaa/1.txt</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">myfile</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mydir</span></span><br><span class="line">    <span class="attr">hostPath:</span></span><br><span class="line">      <span class="comment"># 确保文件所在目录成功创建。</span></span><br><span class="line">      <span class="attr">path:</span> <span class="string">/var/local/aaa</span></span><br><span class="line">      <span class="attr">type:</span> <span class="string">DirectoryOrCreate</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">myfile</span></span><br><span class="line">    <span class="attr">hostPath:</span></span><br><span class="line">      <span class="attr">path:</span> <span class="string">/var/local/aaa/1.txt</span></span><br><span class="line">      <span class="attr">type:</span> <span class="string">FileOrCreate</span></span><br></pre></td></tr></table></figure>



<h3 id="1-1-3-NFS"><a href="#1-1-3-NFS" class="headerlink" title="1.1.3 NFS"></a>1.1.3 NFS</h3><h4 id="1-1-3-1-NFS介绍"><a href="#1-1-3-1-NFS介绍" class="headerlink" title="1.1.3.1 NFS介绍"></a>1.1.3.1 NFS介绍</h4><p><code>nfs</code> 卷能将 NFS (网络文件系统) 挂载到你的 Pod 中。 不像 <code>emptyDir</code> 那样会在删除 Pod 的同时也会被删除，<code>nfs</code> 卷的内容在删除 Pod 时会被保存，卷只是被卸载。 这意味着 <code>nfs</code> 卷可以被预先填充数据，并且这些数据可以在 Pod 之间共享。</p>
<blockquote>
<p><strong>注意：</strong> 在使用 NFS 卷之前，你必须运行自己的 NFS 服务器并将目标 share 导出备用。</p>
</blockquote>
<h4 id="1-1-4-2-NFS配置示例"><a href="#1-1-4-2-NFS配置示例" class="headerlink" title="1.1.4.2 NFS配置示例"></a>1.1.4.2 NFS配置示例</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">test-webserver</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-webserver</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">k8s.gcr.io/test-webserver:latest</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/opt</span></span><br><span class="line">      <span class="attr">name:</span> <span class="string">nfs-volume</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">nfs-volume</span></span><br><span class="line">    <span class="attr">nfs:</span></span><br><span class="line">      <span class="attr">server:</span> <span class="number">192.168</span><span class="number">.0</span><span class="number">.204</span></span><br><span class="line">      <span class="attr">path:</span> <span class="string">/data/nfs/test-dp</span></span><br></pre></td></tr></table></figure>





<h3 id="1-1-4-subPath"><a href="#1-1-4-subPath" class="headerlink" title="1.1.4 subPath"></a>1.1.4 subPath</h3><p>有时，在单个 Pod 中共享卷以供多方使用是很有用的。 <code>volumeMounts.subPath</code> 属性可用于指定所引用的卷内的子路径，而不是其根路径。</p>
<p>下面例子展示了如何配置某包含 LAMP 堆栈（Linux Apache MySQL PHP）的 Pod 使用同一共享卷。 此示例中的 <code>subPath</code> 配置不建议在生产环境中使用。 PHP 应用的代码和相关数据映射到卷的 <code>html</code> 文件夹，MySQL 数据库存储在卷的 <code>mysql</code> 文件夹中：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">my-lamp-site</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">    <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mysql</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">mysql</span></span><br><span class="line">      <span class="attr">env:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">MYSQL_ROOT_PASSWORD</span></span><br><span class="line">        <span class="attr">value:</span> <span class="string">&quot;rootpasswd&quot;</span></span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/var/lib/mysql</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">site-data</span></span><br><span class="line">        <span class="attr">subPath:</span> <span class="string">mysql</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">php</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">php:7.0-apache</span></span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">/var/www/html</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">site-data</span></span><br><span class="line">        <span class="attr">subPath:</span> <span class="string">html</span></span><br><span class="line">    <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">site-data</span></span><br><span class="line">      <span class="attr">persistentVolumeClaim:</span></span><br><span class="line">        <span class="attr">claimName:</span> <span class="string">my-lamp-site-data</span></span><br></pre></td></tr></table></figure>





<h1 id="2-pv-amp-pvc"><a href="#2-pv-amp-pvc" class="headerlink" title="2 pv&amp;pvc"></a>2 pv&amp;pvc</h1><p>存储的管理是一个与计算实例的管理完全不同的问题。PersistentVolume 子系统为用户 和管理员提供了一组 API，将存储如何供应的细节从其如何被使用中抽象出来。 为了实现这点，我们引入了两个新的 API 资源：PersistentVolume 和 PersistentVolumeClaim。</p>
<h2 id="2-1-PV介绍"><a href="#2-1-PV介绍" class="headerlink" title="2.1 PV介绍"></a>2.1 PV介绍</h2><p>持久卷（PersistentVolume，PV）是集群中的一块存储，可以由管理员事先供应，或者 使用<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/storage-classes/">存储类（Storage Class）</a>来动态供应。 持久卷是集群资源，就像节点也是集群资源一样。PV 持久卷和普通的 Volume 一样，也是使用 卷插件来实现的，只是它们拥有独立于任何使用 PV 的 Pod 的生命周期。 此 API 对象中记述了存储的实现细节，无论其背后是 NFS、iSCSI 还是特定于云平台的存储系统。</p>
<h2 id="2-2-PVC介绍"><a href="#2-2-PVC介绍" class="headerlink" title="2.2 PVC介绍"></a>2.2 PVC介绍</h2><p>持久卷申领（PersistentVolumeClaim，PVC）表达的是用户对存储的请求。概念上与 Pod 类似。 Pod 会耗用节点资源，而 PVC 申领会耗用 PV 资源。Pod 可以请求特定数量的资源（CPU 和内存）；同样 PVC 申领也可以请求特定的大小和访问模式 （例如，可以要求 PV 卷能够以 ReadWriteOnce、ReadOnlyMany 或 ReadWriteMany 模式之一来挂载，参见<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/persistent-volumes/#access-modes">访问模式</a>）。</p>
<p>尽管 PersistentVolumeClaim 允许用户消耗抽象的存储资源，常见的情况是针对不同的 问题用户需要的是具有不同属性（如，性能）的 PersistentVolume 卷。 集群管理员需要能够提供不同性质的 PersistentVolume，并且这些 PV 卷之间的差别不 仅限于卷大小和访问模式，同时又不能将卷是如何实现的这些细节暴露给用户。 为了满足这类需求，就有了 <em>存储类（StorageClass）</em> 资源。</p>
<h2 id="2-3-NFS的PV示例"><a href="#2-3-NFS的PV示例" class="headerlink" title="2.3 NFS的PV示例"></a>2.3 NFS的PV示例</h2><p>每个 PV 对象都包含 <code>spec</code> 部分和 <code>status</code> 部分，分别对应卷的规约和状态。 PersistentVolume 对象的名称必须是合法的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/names#dns-subdomain-names">DNS 子域名</a>.</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">PersistentVolume</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">pv0003</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">capacity:</span></span><br><span class="line">    <span class="attr">storage:</span> <span class="string">5Gi</span></span><br><span class="line">  <span class="attr">volumeMode:</span> <span class="string">Filesystem</span></span><br><span class="line">  <span class="attr">accessModes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">ReadWriteOnce</span></span><br><span class="line">  <span class="attr">persistentVolumeReclaimPolicy:</span> <span class="string">Recycle</span></span><br><span class="line">  <span class="attr">storageClassName:</span> <span class="string">slow</span></span><br><span class="line">  <span class="attr">mountOptions:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">hard</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">nfsvers=4.1</span></span><br><span class="line">  <span class="attr">nfs:</span></span><br><span class="line">    <span class="attr">path:</span> <span class="string">/tmp</span></span><br><span class="line">    <span class="attr">server:</span> <span class="number">172.17</span><span class="number">.0</span><span class="number">.2</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>说明：</strong> 在集群中使用持久卷存储通常需要一些特定于具体卷类型的辅助程序。 在这个例子中，PersistentVolume 是 NFS 类型的，因此需要辅助程序 <code>/sbin/mount.nfs</code> 来支持挂载 NFS 文件系统。</p>
</blockquote>
<h3 id="2-3-1-常见字段解析"><a href="#2-3-1-常见字段解析" class="headerlink" title="2.3.1 常见字段解析"></a>2.3.1 常见字段解析</h3><h4 id="2-3-1-1-容量capacity"><a href="#2-3-1-1-容量capacity" class="headerlink" title="2.3.1.1 容量capacity"></a>2.3.1.1 容量<code>capacity</code></h4><p>一般而言，每个 PV 卷都有确定的存储容量。 容量属性是使用 PV 对象的 <code>capacity</code> 属性来设置的。</p>
<p> <code>capacity</code> 字段可以接受的单位:</p>
<ul>
<li>Ei, Pi, Ti, Gi, Mi, Ki</li>
<li>E, P, T, G, M, K, m</li>
</ul>
<p>目前，存储大小是可以设置和请求的唯一资源。 未来可能会包含 IOPS、吞吐量等属性。</p>
<h4 id="2-3-1-2-卷模式volumeModes"><a href="#2-3-1-2-卷模式volumeModes" class="headerlink" title="2.3.1.2 卷模式volumeModes"></a>2.3.1.2 卷模式<code>volumeModes</code></h4><p><strong>FEATURE STATE:</strong> <code>Kubernetes v1.18 [stable]</code></p>
<p>针对 PV 持久卷，Kuberneretes 支持两种卷模式（<code>volumeModes</code>）：<code>Filesystem（文件系统）</code> 和 <code>Block（块）</code>。 <code>volumeMode</code> 是一个可选的 API 参数。 如果该参数被省略，默认的卷模式是 <code>Filesystem</code>。</p>
<p><code>volumeMode</code> 属性设置为 <code>Filesystem</code> 的卷会被 Pod <em>挂载（Mount）</em> 到某个目录。 如果卷的存储来自某块设备而该设备目前为空，Kuberneretes 会在第一次挂载卷之前 在设备上创建文件系统。</p>
<p>你可以将 <code>volumeMode</code> 设置为 <code>Block</code>，以便将卷作为原始块设备来使用。 这类卷以块设备的方式交给 Pod 使用，其上没有任何文件系统。 这种模式对于为 Pod 提供一种使用最快可能方式来访问卷而言很有帮助，Pod 和 卷之间不存在文件系统层。另外，Pod 中运行的应用必须知道如何处理原始块设备。 关于如何在 Pod 中使用 <code>volumeMode: Block</code> 的卷，可参阅 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/persistent-volumes/#raw-block-volume-support">原始块卷支持</a>。</p>
<h4 id="2-3-1-3-访问模式accessModes"><a href="#2-3-1-3-访问模式accessModes" class="headerlink" title="2.3.1.3 访问模式accessModes"></a>2.3.1.3 访问模式<code>accessModes</code></h4><p>PersistentVolume 卷可以用资源提供者所支持的任何方式挂载到宿主系统上。 如下表所示，提供者（驱动）的能力不同，每个 PV 卷的访问模式都会设置为 对应卷所支持的模式值。 例如，NFS 可以支持多个读写客户，但是某个特定的 NFS PV 卷可能在服务器 上以只读的方式导出。每个 PV 卷都会获得自身的访问模式集合，描述的是 特定 PV 卷的能力。</p>
<p>访问模式有：</p>
<ul>
<li>ReadWriteOnce – 卷可以被一个节点以读写方式挂载；</li>
<li>ReadOnlyMany – 卷可以被多个节点以只读方式挂载；</li>
<li>ReadWriteMany – 卷可以被多个节点以读写方式挂载。</li>
</ul>
<p>在命令行接口（CLI）中，访问模式也使用以下缩写形式：</p>
<ul>
<li>RWO - ReadWriteOnce</li>
<li>ROX - ReadOnlyMany</li>
<li>RWX - ReadWriteMany</li>
</ul>
<blockquote>
<p><strong>重要提醒！</strong> 每个卷只能同一时刻只能以一种访问模式挂载，即使该卷能够支持 多种访问模式。例如，一个 GCEPersistentDisk 卷可以被某节点以 ReadWriteOnce 模式挂载，或者被多个节点以 ReadOnlyMany 模式挂载，但不可以同时以两种模式 挂载。</p>
</blockquote>
<h4 id="2-3-1-4-类storageClassName"><a href="#2-3-1-4-类storageClassName" class="headerlink" title="2.3.1.4 类storageClassName"></a>2.3.1.4 类<code>storageClassName</code></h4><p>每个 PV 可以属于某个类（Class），通过将其 <code>storageClassName</code> 属性设置为某个 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/storage-classes/">StorageClass</a> 的名称来指定。 特定类的 PV 卷只能绑定到请求该类存储卷的 PVC 申领。 未设置 <code>storageClassName</code> 的 PV 卷没有类设定，只能绑定到那些没有指定特定 存储类的 PVC 申领。</p>
<h4 id="2-3-1-5-回收策略-persistentVolumeReclaimPolicy"><a href="#2-3-1-5-回收策略-persistentVolumeReclaimPolicy" class="headerlink" title="2.3.1.5 回收策略 persistentVolumeReclaimPolicy"></a>2.3.1.5 回收策略 <code>persistentVolumeReclaimPolicy</code></h4><p>目前的回收策略有：</p>
<ul>
<li>Retain – 手动回收</li>
<li>Recycle – 基本擦除 (<code>rm -rf /thevolume/*</code>)</li>
<li>Delete – 诸如 AWS EBS、GCE PD、Azure Disk 或 OpenStack Cinder 卷这类关联存储资产也被删除</li>
</ul>
<p>目前，仅 NFS 和 HostPath 支持回收（Recycle）。 AWS EBS、GCE PD、Azure Disk 和 Cinder 卷都支持删除（Delete）。</p>
<h2 id="2-4-PV的不同状态阶段"><a href="#2-4-PV的不同状态阶段" class="headerlink" title="2.4 PV的不同状态阶段"></a>2.4 PV的不同状态阶段</h2><p>每个卷会处于以下阶段（Phase）之一：</p>
<ul>
<li>Available（可用）– 卷是一个空闲资源，尚未绑定到任何申领；</li>
<li>Bound（已绑定）– 该卷已经绑定到某申领；</li>
<li>Released（已释放）– 所绑定的申领已被删除，但是资源尚未被集群回收；</li>
<li>Failed（失败）– 卷的自动回收操作失败。</li>
</ul>
<p>命令行接口能够显示绑定到某 PV 卷的 PVC 对象。</p>
<h2 id="2-5-PVC示例"><a href="#2-5-PVC示例" class="headerlink" title="2.5 PVC示例"></a>2.5 PVC示例</h2><p>每个 PVC 对象都有 <code>spec</code> 和 <code>status</code> 部分，分别对应申领的规约和状态。 PersistentVolumeClaim 对象的名称必须是合法的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/names#dns-subdomain-names">DNS 子域名</a>.</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">PersistentVolumeClaim</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">myclaim</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">accessModes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">ReadWriteOnce</span></span><br><span class="line">  <span class="attr">volumeMode:</span> <span class="string">Filesystem</span></span><br><span class="line">  <span class="attr">resources:</span></span><br><span class="line">    <span class="attr">requests:</span></span><br><span class="line">      <span class="attr">storage:</span> <span class="string">8Gi</span></span><br><span class="line">  <span class="attr">storageClassName:</span> <span class="string">slow</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">release:</span> <span class="string">&quot;stable&quot;</span></span><br><span class="line">    <span class="attr">matchExpressions:</span></span><br><span class="line">      <span class="bullet">-</span> &#123;<span class="attr">key:</span> <span class="string">environment</span>, <span class="attr">operator:</span> <span class="string">In</span>, <span class="attr">values:</span> [<span class="string">dev</span>]&#125;</span><br></pre></td></tr></table></figure>



<h3 id="2-5-1-常见字段解析"><a href="#2-5-1-常见字段解析" class="headerlink" title="2.5.1 常见字段解析"></a>2.5.1 常见字段解析</h3><h4 id="2-5-1-1-访问模式accessModes"><a href="#2-5-1-1-访问模式accessModes" class="headerlink" title="2.5.1.1 访问模式accessModes"></a>2.5.1.1 访问模式<code>accessModes</code></h4><p>申领在请求具有特定访问模式的存储时，使用与卷相同的访问模式约定。</p>
<h4 id="2-5-1-2-卷模式volumeMode"><a href="#2-5-1-2-卷模式volumeMode" class="headerlink" title="2.5.1.2 卷模式volumeMode"></a>2.5.1.2 卷模式<code>volumeMode</code></h4><p>申领使用与卷相同的约定来表明是将卷作为文件系统还是块设备来使用。</p>
<h4 id="2-5-1-3-资源resources"><a href="#2-5-1-3-资源resources" class="headerlink" title="2.5.1.3 资源resources"></a>2.5.1.3 资源<code>resources</code></h4><p>申领和 Pod 一样，也可以请求特定数量的资源。在这个上下文中，请求的资源是存储。 卷和申领都使用相同的 <a target="_blank" rel="noopener" href="https://git.k8s.io/community/contributors/design-proposals/scheduling/resources.md">资源模型</a>。</p>
<h4 id="2-5-1-4-选择算符"><a href="#2-5-1-4-选择算符" class="headerlink" title="2.5.1.4 选择算符"></a>2.5.1.4 选择算符</h4><p>申领可以设置<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/labels/#label-selectors">标签选择算符</a> 来进一步过滤卷集合。只有标签与选择算符相匹配的卷能够绑定到申领上。 选择算符包含两个字段：</p>
<ul>
<li><code>matchLabels</code> - 卷必须包含带有此值的标签</li>
<li><code>matchExpressions</code> - 通过设定键（key）、值列表和操作符（operator） 来构造的需求。合法的操作符有 In、NotIn、Exists 和 DoesNotExist。</li>
</ul>
<p>来自 <code>matchLabels</code> 和 <code>matchExpressions</code> 的所有需求都按逻辑与的方式组合在一起。 这些需求都必须被满足才被视为匹配。</p>
<h4 id="2-5-1-5-类storageClassName"><a href="#2-5-1-5-类storageClassName" class="headerlink" title="2.5.1.5 类storageClassName"></a>2.5.1.5 类<code>storageClassName</code></h4><p>申领可以通过为 <code>storageClassName</code> 属性设置 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/storage-classes/">StorageClass</a> 的名称来请求特定的存储类。 只有所请求的类的 PV 卷，即 <code>storageClassName</code> 值与 PVC 设置相同的 PV 卷， 才能绑定到 PVC 申领。</p>
<p>PVC 申领不必一定要请求某个类。如果 PVC 的 <code>storageClassName</code> 属性值设置为 <code>&quot;&quot;</code>， 则被视为要请求的是没有设置存储类的 PV 卷，因此这一 PVC 申领只能绑定到未设置 存储类的 PV 卷（未设置注解或者注解值为 <code>&quot;&quot;</code> 的 PersistentVolume（PV）对象在系统中不会被删除，因为这样做可能会引起数据丢失。 未设置 <code>storageClassName</code> 的 PVC 与此大不相同，也会被集群作不同处理。 具体筛查方式取决于 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/access-authn-authz/admission-controllers/#defaultstorageclass"><code>DefaultStorageClass</code> 准入控制器插件</a> 是否被启用。</p>
<ul>
<li>如果准入控制器插件被启用，则管理员可以设置一个默认的 StorageClass。 所有未设置 <code>storageClassName</code> 的 PVC 都只能绑定到隶属于默认存储类的 PV 卷。 设置默认 StorageClass 的工作是通过将对应 StorageClass 对象的注解 <code>storageclass.kubernetes.io/is-default-class</code> 赋值为 <code>true</code> 来完成的。 如果管理员未设置默认存储类，集群对 PVC 创建的处理方式与未启用准入控制器插件 时相同。如果设定的默认存储类不止一个，准入控制插件会禁止所有创建 PVC 操作。</li>
<li>如果准入控制器插件被关闭，则不存在默认 StorageClass 的说法。 所有未设置 <code>storageClassName</code> 的 PVC 都只能绑定到未设置存储类的 PV 卷。 在这种情况下，未设置 <code>storageClassName</code> 的 PVC 与 <code>storageClassName</code> 设置未 <code>&quot;&quot;</code> 的 PVC 的处理方式相同。</li>
</ul>
<p>取决于安装方法，默认的 StorageClass 可能在集群安装期间由插件管理器（Addon Manager）部署到集群中。</p>
<p>当某 PVC 除了请求 StorageClass 之外还设置了 <code>selector</code>，则这两种需求会按 逻辑与关系处理：只有隶属于所请求类且带有所请求标签的 PV 才能绑定到 PVC。</p>
<blockquote>
<p><strong>说明：</strong> 目前，设置了非空 <code>selector</code> 的 PVC 对象无法让集群为其动态供应 PV 卷。</p>
</blockquote>
<h2 id="2-6-挂载PVC"><a href="#2-6-挂载PVC" class="headerlink" title="2.6 挂载PVC"></a>2.6 挂载PVC</h2><p>Pod 将PVC作为卷来使用，并藉此访问存储资源。 PVC必须位于使用它的 Pod 所在同一命名空间内。 集群在 Pod 的命名空间中查找PVC，并使用它来获得PVC所使用的 PV 卷。 之后，卷会被挂载到宿主上并挂载到 Pod 中。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mypod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">myfrontend</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">nginx</span></span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">mountPath:</span> <span class="string">&quot;/var/www/html&quot;</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">mypd</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypd</span></span><br><span class="line">      <span class="attr">persistentVolumeClaim:</span></span><br><span class="line">        <span class="attr">claimName:</span> <span class="string">myclaim</span></span><br></pre></td></tr></table></figure>

<h3 id="2-6-1-关于命名空间的说明"><a href="#2-6-1-关于命名空间的说明" class="headerlink" title="2.6.1 关于命名空间的说明"></a>2.6.1 关于命名空间的说明</h3><p>PersistentVolume 卷的绑定是排他性的。 由于 PersistentVolumeClaim 是命名空间作用域的对象，使用 “Many” 模式（<code>ROX</code>、<code>RWX</code>）来挂载PVC的操作只能在同一命名空间内进行。</p>
<h2 id="3-关于PVC和PV的异常汇总"><a href="#3-关于PVC和PV的异常汇总" class="headerlink" title="3 关于PVC和PV的异常汇总"></a>3 关于PVC和PV的异常汇总</h2><h3 id="3-1-创建PVC之后，一直绑定不上PV（Pending）"><a href="#3-1-创建PVC之后，一直绑定不上PV（Pending）" class="headerlink" title="3.1 创建PVC之后，一直绑定不上PV（Pending）"></a>3.1 创建PVC之后，一直绑定不上PV（Pending）</h3><ol>
<li><p>PVC的空间申请大小大于PV的大小</p>
</li>
<li><p>PVC的StorageClassName没有和PV的一致</p>
</li>
<li><p>PVC的accessModes和PV的不一致</p>
</li>
</ol>
<h3 id="3-2-创建挂载了PVC的Pod之后，一直处于Pending状态"><a href="#3-2-创建挂载了PVC的Pod之后，一直处于Pending状态" class="headerlink" title="3.2 创建挂载了PVC的Pod之后，一直处于Pending状态"></a>3.2 创建挂载了PVC的Pod之后，一直处于Pending状态</h3><ol>
<li><p>PVC没有被创建成功，或者被创建</p>
</li>
<li><p>PVC和Pod不在同一个Namespace</p>
</li>
</ol>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color2">卷</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color4">持久卷</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color2">pv&pvc</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8Volumes&pv&pvc/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/kubeadm安装k8s高可用集群" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/kubeadm%E5%AE%89%E8%A3%85k8s%E9%AB%98%E5%8F%AF%E7%94%A8%E9%9B%86%E7%BE%A4/">kubeadm安装1.19高可用集群(生产级)</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/kubeadm%E5%AE%89%E8%A3%85k8s%E9%AB%98%E5%8F%AF%E7%94%A8%E9%9B%86%E7%BE%A4/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="kubeadm安装k8s-1-19集群"><a href="#kubeadm安装k8s-1-19集群" class="headerlink" title="kubeadm安装k8s 1.19集群"></a>kubeadm安装k8s 1.19集群</h2><h3 id="1-基本说明"><a href="#1-基本说明" class="headerlink" title="1.   基本说明"></a>1.   基本说明</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">系统：centos7  或 centos8</span><br><span class="line"></span><br><span class="line">k8s: 1.19</span><br></pre></td></tr></table></figure>

<p><strong>生产环境中，建议使用小版本大于5的Kubernetes版本，比如1.19.5以后的才可用于生产环境。</strong></p>
<h3 id="2-基本环境"><a href="#2-基本环境" class="headerlink" title="2. 基本环境"></a>2. 基本环境</h3><h4 id="2-1-主机信息"><a href="#2-1-主机信息" class="headerlink" title="2.1 主机信息"></a>2.1 主机信息</h4><table>
<thead>
<tr>
<th>hostname</th>
<th>ip</th>
<th>配置</th>
</tr>
</thead>
<tbody><tr>
<td>k8s-master01</td>
<td>192.168.31.120</td>
<td>2C2G 30G</td>
</tr>
<tr>
<td>k8s-master02</td>
<td>192.168.31.121</td>
<td>2C2G 30G</td>
</tr>
<tr>
<td>k8s-master03</td>
<td>192.168.31.122</td>
<td>2C2G 30G</td>
</tr>
<tr>
<td>k8s-node01</td>
<td>192.168.31.123</td>
<td>2C2G 30G</td>
</tr>
<tr>
<td>k8s-node02</td>
<td>192.168.31.124</td>
<td>2C2G 30G</td>
</tr>
<tr>
<td>k8s-master-lb</td>
<td>192.168.31.125</td>
<td>VIP 虚IP不占用机器资源</td>
</tr>
</tbody></table>
<h4 id="2-2-虚拟机环境"><a href="#2-2-虚拟机环境" class="headerlink" title="2.2 虚拟机环境"></a>2.2 虚拟机环境</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">VmWare 15.5.6 build-16341506</span><br></pre></td></tr></table></figure>



<h4 id="2-3-基础环境配置"><a href="#2-3-基础环境配置" class="headerlink" title="2.3 基础环境配置"></a>2.3 基础环境配置</h4><ul>
<li><p>配置所有节点hosts文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt; EOF &gt;&gt; &#x2F;etc&#x2F;hosts</span><br><span class="line">192.168.31.120 k8s-master01 </span><br><span class="line">192.168.31.121 k8s-master02 </span><br><span class="line">192.168.31.122 k8s-master03 </span><br><span class="line">192.168.31.123 k8s-node01 </span><br><span class="line">192.168.31.124 k8s-node02  </span><br><span class="line">192.168.31.125 k8s-master-lb</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>所有节点配置，CentOS 7安装yum源</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo</span><br><span class="line"></span><br><span class="line">wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo</span><br><span class="line"></span><br><span class="line">yum clean all &amp;&amp; yum makecache</span><br><span class="line"></span><br><span class="line">yum install -y yum-utils device-mapper-persistent-data lvm2</span><br><span class="line"></span><br><span class="line">yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">cat &lt;&lt;EOF &gt; /etc/yum.repos.d/kubernetes.repo</span><br><span class="line">[kubernetes]</span><br><span class="line">name=Kubernetes</span><br><span class="line">baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/</span><br><span class="line">enabled=1</span><br><span class="line">gpgcheck=1</span><br><span class="line">repo_gpgcheck=1</span><br><span class="line">gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg</span><br><span class="line">EOF</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>centos8安装如下</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">curl -o &#x2F;etc&#x2F;yum.repos.d&#x2F;CentOS-Base.repo http:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;repo&#x2F;Centos-8.repo</span><br><span class="line"></span><br><span class="line">wget -O &#x2F;etc&#x2F;yum.repos.d&#x2F;epel.repo http:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;repo&#x2F;epel-8.repo</span><br><span class="line"></span><br><span class="line">yum makecache</span><br><span class="line"></span><br><span class="line">yum install -y yum-utils device-mapper-persistent-data lvm2</span><br><span class="line"></span><br><span class="line">yum-config-manager --add-repo https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;docker-ce&#x2F;linux&#x2F;centos&#x2F;docker-ce.repo</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">cat &lt;&lt;EOF &gt; &#x2F;etc&#x2F;yum.repos.d&#x2F;kubernetes.repo</span><br><span class="line">[kubernetes]</span><br><span class="line">name&#x3D;Kubernetes</span><br><span class="line">baseurl&#x3D;https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;kubernetes&#x2F;yum&#x2F;repos&#x2F;kubernetes-el7-x86_64&#x2F;</span><br><span class="line">enabled&#x3D;1</span><br><span class="line">gpgcheck&#x3D;1</span><br><span class="line">repo_gpgcheck&#x3D;1</span><br><span class="line">gpgkey&#x3D;https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;kubernetes&#x2F;yum&#x2F;doc&#x2F;yum-key.gpg https:&#x2F;&#x2F;mirrors.aliyun.com&#x2F;kubernetes&#x2F;yum&#x2F;doc&#x2F;rpm-package-key.gpg</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>





</li>
</ul>
<ul>
<li><p>所有节点关闭firewalld 、dnsmasq、selinux(CentOS7需要关闭NetworkManager，CentOS8不需要)</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">systemctl stop firewalld &amp;&amp; systemctl disable --now firewalld;\</span><br><span class="line">systemctl stop dnsmasq systemctl disable --now dnsmasq ;\</span><br><span class="line"></span><br><span class="line">systemctl stop NetworkManager &amp;&amp; systemctl disable NetworkManager;\</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">setenforce 0; \</span><br><span class="line">sed -i &#x27;s/SELINUX=enforcing/SELINUX=disabled/&#x27; /etc/sysconfig/selinux</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有节点关闭swap分区，fstab注释swap</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">swapoff -a &amp;&amp; sysctl -w vm.swappiness&#x3D;0</span><br><span class="line"></span><br><span class="line">sed -ri &#39;s&#x2F;(^\&#x2F;.+swap.+$)&#x2F;#\1&#x2F;&#39; &#x2F;etc&#x2F;fstab</span><br><span class="line"></span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>所有节点同步时间</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">rpm -ivh http:&#x2F;&#x2F;mirrors.wlnmp.com&#x2F;centos&#x2F;wlnmp-release-centos.noarch.rpm &amp;&amp; \</span><br><span class="line">yum install wntp -y &amp;&amp; \</span><br><span class="line">ln -sf &#x2F;usr&#x2F;share&#x2F;zoneinfo&#x2F;Asia&#x2F;Shanghai &#x2F;etc&#x2F;localtime &amp;&amp; \</span><br><span class="line">echo &#39;Asia&#x2F;Shanghai&#39; &gt; &#x2F;etc&#x2F;timezone &amp;&amp; \</span><br><span class="line">ntpdate time2.aliyun.com</span><br></pre></td></tr></table></figure>
</li>
<li><p>Master01节点生成ssh key</p>
<p>一路回车即可</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ssh-keygen -t rsa</span><br></pre></td></tr></table></figure>
</li>
<li><p>设置ulimit</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">ulimit -SHn 65535  &amp;&amp; \</span><br><span class="line">cat &lt;&lt; EOF &gt;&gt; &#x2F;etc&#x2F;security&#x2F;limits.conf</span><br><span class="line">* soft nproc 65535</span><br><span class="line">* hard nproc 65535</span><br><span class="line">* soft nofile 65535</span><br><span class="line">* hard nofile 65535</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>Master01配置免密码登录其他节点</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02; \</span><br><span class="line"><span class="keyword">do</span> \</span><br><span class="line">	ssh-copy-id -i .ssh/id_rsa.pub <span class="variable">$i</span>; \</span><br><span class="line">	<span class="keyword">done</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>克隆代码</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">git clone https:&#x2F;&#x2F;github.com&#x2F;dotbalo&#x2F;k8s-ha-install.git </span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="3-基本组件安装"><a href="#3-基本组件安装" class="headerlink" title="3. 基本组件安装"></a>3. 基本组件安装</h3><h4 id="3-1-系统升级"><a href="#3-1-系统升级" class="headerlink" title="3.1 系统升级"></a>3.1 系统升级</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum update -y</span><br></pre></td></tr></table></figure>



<h4 id="3-2-内核升级"><a href="#3-2-内核升级" class="headerlink" title="3.2 内核升级"></a>3.2 内核升级</h4><p>低于内核4.18需要升级内核版本，防止出现docker bug</p>
<h5 id="3-2-3-CentOs7-升级内核"><a href="#3-2-3-CentOs7-升级内核" class="headerlink" title="3.2.3  CentOs7 升级内核"></a>3.2.3  CentOs7 升级内核</h5><ul>
<li><p>导入源</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">rpm --import https:&#x2F;&#x2F;www.elrepo.org&#x2F;RPM-GPG-KEY-elrepo.org</span><br><span class="line"></span><br><span class="line">rpm -Uvh http:&#x2F;&#x2F;www.elrepo.org&#x2F;elrepo-release-7.0-2.el7.elrepo.noarch.rpm</span><br></pre></td></tr></table></figure>
</li>
<li><p>安装最新版</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum --enablerepo&#x3D;elrepo-kernel install kernel-ml kernel-ml-devel -y</span><br></pre></td></tr></table></figure>
</li>
<li><p>重启</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">reboot</span><br></pre></td></tr></table></figure>
</li>
<li><p>更改内核顺序</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">grub2-set-default  0 &amp;&amp; grub2-mkconfig -o &#x2F;etc&#x2F;grub2.cfg &amp;&amp; grubby --args&#x3D;&quot;user_namespace.enable&#x3D;1&quot; --update-kernel&#x3D;&quot;$(grubby --default-kernel)&quot; &amp;&amp; reboot</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h5 id="3-2-4-CentOs8升级内核"><a href="#3-2-4-CentOs8升级内核" class="headerlink" title="3.2.4 CentOs8升级内核"></a>3.2.4 CentOs8升级内核</h5><ul>
<li><p>Cen可以采用dnf升级，也可使用CentOs7同样步骤升级（使用上述步骤注意elrepo-release-8.1版本）</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">rpm --import https:&#x2F;&#x2F;www.elrepo.org&#x2F;RPM-GPG-KEY-elrepo.org</span><br><span class="line"></span><br><span class="line">yum install https:&#x2F;&#x2F;www.elrepo.org&#x2F;elrepo-release-8.el8.elrepo.noarch.rpm</span><br><span class="line"></span><br><span class="line">dnf --disablerepo&#x3D;\* --enablerepo&#x3D;elrepo-kernel -y install kernel-ml kernel-ml-devel</span><br><span class="line"></span><br><span class="line">grubby --default-kernel &amp;&amp; reboot</span><br><span class="line"></span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="4-k8s组件安装"><a href="#4-k8s组件安装" class="headerlink" title="4. k8s组件安装"></a>4. k8s组件安装</h3><ul>
<li><p>所有节点安装ipvs</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum install ipvsadm ipset sysstat conntrack libseccomp -y</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有节点配置ipvs模块</p>
<p>在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack</p>
<p>以下为4.19+版本</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">modprobe -- ip_vs  &amp;&amp; \</span><br><span class="line">modprobe -- ip_vs_rr &amp;&amp; \</span><br><span class="line">modprobe -- ip_vs_wrr &amp;&amp; \</span><br><span class="line">modprobe -- ip_vs_sh &amp;&amp; \</span><br><span class="line">modprobe -- nf_conntrack</span><br></pre></td></tr></table></figure>
</li>
<li><p>设置模块开机自启动</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt; EOF &gt;&gt; /etc/modules-load.d/ipvs.conf</span><br><span class="line">ip_vs</span><br><span class="line">ip_vs_rr</span><br><span class="line">ip_vs_wrr</span><br><span class="line">ip_vs_sh</span><br><span class="line">nf_conntrack</span><br><span class="line">ip_tables</span><br><span class="line">ip_set</span><br><span class="line">xt_set</span><br><span class="line">ipt_set</span><br><span class="line">ipt_rpfilter</span><br><span class="line">ipt_REJECT</span><br><span class="line">ipip</span><br><span class="line">EOF </span><br></pre></td></tr></table></figure>

<p>开机加载</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">systemctl enable --now systemd-modules-load.service</span><br></pre></td></tr></table></figure>

<p>检查是否加载：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">lsmod | grep -e ip_vs -e nf_conntrack_ipv4</span><br></pre></td></tr></table></figure>
</li>
<li><p>开启一些k8s集群中必须的内核参数，所有节点配置k8s内核</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF &gt; &#x2F;etc&#x2F;sysctl.d&#x2F;k8s.conf</span><br><span class="line">net.ipv4.ip_forward &#x3D; 1</span><br><span class="line">net.bridge.bridge-nf-call-iptables &#x3D; 1</span><br><span class="line">fs.may_detach_mounts &#x3D; 1</span><br><span class="line">vm.overcommit_memory&#x3D;1</span><br><span class="line">vm.panic_on_oom&#x3D;0</span><br><span class="line">fs.inotify.max_user_watches&#x3D;89100</span><br><span class="line">fs.file-max&#x3D;52706963</span><br><span class="line">fs.nr_open&#x3D;52706963</span><br><span class="line">net.netfilter.nf_conntrack_max&#x3D;2310720</span><br><span class="line"></span><br><span class="line">net.ipv4.tcp_keepalive_time &#x3D; 600</span><br><span class="line">net.ipv4.tcp_keepalive_probes &#x3D; 3</span><br><span class="line">net.ipv4.tcp_keepalive_intvl &#x3D;15</span><br><span class="line">net.ipv4.tcp_max_tw_buckets &#x3D; 36000</span><br><span class="line">net.ipv4.tcp_tw_reuse &#x3D; 1</span><br><span class="line">net.ipv4.tcp_max_orphans &#x3D; 327680</span><br><span class="line">net.ipv4.tcp_orphan_retries &#x3D; 3</span><br><span class="line">net.ipv4.tcp_syncookies &#x3D; 1</span><br><span class="line">net.ipv4.tcp_max_syn_backlog &#x3D; 16384</span><br><span class="line">net.ipv4.ip_conntrack_max &#x3D; 65536</span><br><span class="line">net.ipv4.tcp_max_syn_backlog &#x3D; 16384</span><br><span class="line">net.ipv4.tcp_timestamps &#x3D; 0</span><br><span class="line">net.core.somaxconn &#x3D; 16384</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">sysctl --system</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有节点安装最新版Docker</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum install docker-ce -y</span><br></pre></td></tr></table></figure>

<p><strong>一般而言docker版本需要根据kubernetes的版本变更日志确定安装版本，并不是最新的就是最合适的</strong></p>
<p><strong>安装完docker，可以使用docker info查看是否有报错，出现报错需要及时解决</strong></p>
</li>
</ul>
<p>  <strong>centos8安装docker时可能会报错需要安装新版containerd</strong></p>
  <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">wget https:&#x2F;&#x2F;download.docker.com&#x2F;linux&#x2F;centos&#x2F;7&#x2F;x86_64&#x2F;stable&#x2F;Packages&#x2F;containerd.io-1.2.13-3.2.el7.x86_64.rpm</span><br><span class="line"></span><br><span class="line">yum install containerd.io-1.2.13-3.2.el7.x86_64.rpm -y</span><br></pre></td></tr></table></figure>



<ul>
<li><p>由于新版kubelet建议使用systemd，所以可以把docker的CgroupDriver改成systemd</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">mkdir -p &#x2F;etc&#x2F;docker &amp;&amp; \</span><br><span class="line">cat &gt; &#x2F;etc&#x2F;docker&#x2F;daemon.json &lt;&lt;EOF</span><br><span class="line">&#123;</span><br><span class="line">&quot;registry-mirrors&quot;: [</span><br><span class="line">    &quot;https:&#x2F;&#x2F;registry.docker-cn.com&quot;,</span><br><span class="line">    &quot;http:&#x2F;&#x2F;hub-mirror.c.163.com&quot;,</span><br><span class="line">    &quot;https:&#x2F;&#x2F;docker.mirrors.ustc.edu.cn&quot;</span><br><span class="line">  ],</span><br><span class="line">  &quot;exec-opts&quot;: [&quot;native.cgroupdriver&#x3D;systemd&quot;]</span><br><span class="line">&#125;</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>所有节点开启Docker并设置开机自启动</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">systemctl daemon-reload &amp;&amp; systemctl enable --now docker</span><br></pre></td></tr></table></figure>
</li>
<li><p>安装k8s组件，查看版本信息</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum list kubeadm.x86_64 --showduplicates | sort -r</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有节点安装最新版本kubeadm</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum install kubeadm -y</span><br></pre></td></tr></table></figure>

<p>或者安装指定版本组件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum install -y kubeadm-1.12.3-0.x86_64 kubectl-1.12.3-0.x86_64 kubelet-1.12.3-0.x86_64</span><br></pre></td></tr></table></figure>
</li>
<li><p>默认配置的pause镜像使用gcr.io仓库，国内可能无法访问，所以这里配置Kubelet使用阿里云的pause镜像：</p>
<p>注意： –cgroup-driver可以填写为 $DOCKER_CGROUPS 自动获取docker的cgroupDriver，或者直接填写与docker配置相同的cgroupDriver</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">cat &gt;&#x2F;etc&#x2F;sysconfig&#x2F;kubelet&lt;&lt;EOF</span><br><span class="line">KUBELET_EXTRA_ARGS&#x3D;&quot;--cgroup-driver&#x3D;systemd --pod-infra-container-image&#x3D;registry.cn-hangzhou.aliyuncs.com&#x2F;google_containers&#x2F;pause-amd64:3.1&quot;</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>
</li>
<li><p>设置Kubelet开机自启动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">systemctl daemon-reload &amp;&amp; \</span><br><span class="line">systemctl enable --now kubelet</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="5-高可用组件安装"><a href="#5-高可用组件安装" class="headerlink" title="5. 高可用组件安装"></a>5. 高可用组件安装</h3><ul>
<li><p>所有Master节点通过yum安装HAProxy和KeepAlived</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum install keepalived haproxy -y</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有Master节点配置HAProxy（详细配置参考HAProxy文档，所有Master节点的HAProxy配置相同）：</p>
<p><strong>注意修改其中的IP地址</strong></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br></pre></td><td class="code"><pre><span class="line">mkdir &#x2F;etc&#x2F;haproxy &amp;&amp; \</span><br><span class="line"></span><br><span class="line">cat &lt;&lt; EOF &gt;&#x2F;etc&#x2F;haproxy&#x2F;haproxy.cfg </span><br><span class="line">global</span><br><span class="line">  maxconn  2000</span><br><span class="line">  ulimit-n  16384</span><br><span class="line">  log  127.0.0.1 local0 err</span><br><span class="line">  stats timeout 30s</span><br><span class="line"></span><br><span class="line">defaults</span><br><span class="line">  log global</span><br><span class="line">  mode  http</span><br><span class="line">  option  httplog</span><br><span class="line">  timeout connect 5000</span><br><span class="line">  timeout client  50000</span><br><span class="line">  timeout server  50000</span><br><span class="line">  timeout http-request 15s</span><br><span class="line">  timeout http-keep-alive 15s</span><br><span class="line"></span><br><span class="line">frontend monitor-in</span><br><span class="line">  bind *:33305</span><br><span class="line">mode http</span><br><span class="line">  option httplog</span><br><span class="line">  monitor-uri &#x2F;monitor</span><br><span class="line"></span><br><span class="line">listen stats</span><br><span class="line">  bind    *:8006</span><br><span class="line">  mode    http</span><br><span class="line">  stats   enable</span><br><span class="line">  stats   hide-version</span><br><span class="line">  stats   uri       &#x2F;stats</span><br><span class="line">  stats   refresh   30s</span><br><span class="line">  stats   realm     Haproxy\ Statistics</span><br><span class="line">  stats   auth      admin:admin</span><br><span class="line"></span><br><span class="line">frontend k8s-master</span><br><span class="line">  bind 0.0.0.0:16443</span><br><span class="line">  bind 127.0.0.1:16443</span><br><span class="line">  mode tcp</span><br><span class="line">  option tcplog</span><br><span class="line">  tcp-request inspect-delay 5s</span><br><span class="line">  default_backend k8s-master</span><br><span class="line"></span><br><span class="line">backend k8s-master</span><br><span class="line">  mode tcp</span><br><span class="line">  option tcplog</span><br><span class="line">  option tcp-check</span><br><span class="line">  balance roundrobin</span><br><span class="line">  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100</span><br><span class="line">  server k8s-master01 192.168.31.120:6443  check</span><br><span class="line">  server k8s-master02 192.168.31.121:6443  check</span><br><span class="line">  server k8s-master03 192.168.31.122:6443  check</span><br><span class="line"></span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>Master01节点的keepalived配置</p>
<p>需要注意：interface、mcast_src_ip、virtual_router_id、virtual_ipaddress配置项</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt; &#x2F;etc&#x2F;keepalived&#x2F;keepalived.conf </span><br><span class="line">! Configuration File for keepalived</span><br><span class="line">global_defs &#123;</span><br><span class="line">    router_id LVS_DEVEL</span><br><span class="line">&#125;</span><br><span class="line">vrrp_script chk_apiserver &#123;</span><br><span class="line">    script &quot;&#x2F;etc&#x2F;keepalived&#x2F;check_apiserver.sh&quot;</span><br><span class="line">    interval 2</span><br><span class="line">    weight -5</span><br><span class="line">    fall 3  </span><br><span class="line">    rise 2</span><br><span class="line">&#125;</span><br><span class="line">vrrp_instance VI_1 &#123;</span><br><span class="line">    state MASTER</span><br><span class="line">    interface ens33</span><br><span class="line">    mcast_src_ip 192.168.31.120</span><br><span class="line">    virtual_router_id 51</span><br><span class="line">    priority 100</span><br><span class="line">    advert_int 2</span><br><span class="line">    authentication &#123;</span><br><span class="line">        auth_type PASS</span><br><span class="line">        auth_pass K8SHA_KA_AUTH</span><br><span class="line">    &#125;</span><br><span class="line">    virtual_ipaddress &#123;</span><br><span class="line">        192.168.31.125</span><br><span class="line">    &#125;</span><br><span class="line">#    track_script &#123;</span><br><span class="line">#       chk_apiserver</span><br><span class="line">#    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>
</li>
<li><p>Master02节点的keepalived配置</p>
<p>需要注意：interface、mcast_src_ip、virtual_router_id、virtual_ipaddress配置项</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt; &#x2F;etc&#x2F;keepalived&#x2F;keepalived.conf </span><br><span class="line">! Configuration File for keepalived</span><br><span class="line">global_defs &#123;</span><br><span class="line">    router_id LVS_DEVEL</span><br><span class="line">&#125;</span><br><span class="line">vrrp_script chk_apiserver &#123;</span><br><span class="line">    script &quot;&#x2F;etc&#x2F;keepalived&#x2F;check_apiserver.sh&quot;</span><br><span class="line">    interval 2</span><br><span class="line">    weight -5</span><br><span class="line">    fall 3  </span><br><span class="line">    rise 2</span><br><span class="line">&#125;</span><br><span class="line">vrrp_instance VI_1 &#123;</span><br><span class="line">    state MASTER</span><br><span class="line">    interface ens33</span><br><span class="line">    mcast_src_ip 192.168.31.121</span><br><span class="line">    virtual_router_id 51</span><br><span class="line">    priority 100</span><br><span class="line">    advert_int 2</span><br><span class="line">    authentication &#123;</span><br><span class="line">        auth_type PASS</span><br><span class="line">        auth_pass K8SHA_KA_AUTH</span><br><span class="line">    &#125;</span><br><span class="line">    virtual_ipaddress &#123;</span><br><span class="line">        192.168.31.125</span><br><span class="line">    &#125;</span><br><span class="line">#    track_script &#123;</span><br><span class="line">#       chk_apiserver</span><br><span class="line">#    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>
</li>
<li><p>Master03节点的keepalived配置</p>
<p>需要注意：interface、mcast_src_ip、virtual_router_id、virtual_ipaddress配置项</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt; &#x2F;etc&#x2F;keepalived&#x2F;keepalived.conf </span><br><span class="line">! Configuration File for keepalived</span><br><span class="line">global_defs &#123;</span><br><span class="line">    router_id LVS_DEVEL</span><br><span class="line">&#125;</span><br><span class="line">vrrp_script chk_apiserver &#123;</span><br><span class="line">    script &quot;&#x2F;etc&#x2F;keepalived&#x2F;check_apiserver.sh&quot;</span><br><span class="line">    interval 2</span><br><span class="line">    weight -5</span><br><span class="line">    fall 3  </span><br><span class="line">    rise 2</span><br><span class="line">&#125;</span><br><span class="line">vrrp_instance VI_1 &#123;</span><br><span class="line">    state MASTER</span><br><span class="line">    interface ens33</span><br><span class="line">    mcast_src_ip 192.168.31.122</span><br><span class="line">    virtual_router_id 51</span><br><span class="line">    priority 100</span><br><span class="line">    advert_int 2</span><br><span class="line">    authentication &#123;</span><br><span class="line">        auth_type PASS</span><br><span class="line">        auth_pass K8SHA_KA_AUTH</span><br><span class="line">    &#125;</span><br><span class="line">    virtual_ipaddress &#123;</span><br><span class="line">        192.168.31.125</span><br><span class="line">    &#125;</span><br><span class="line">#    track_script &#123;</span><br><span class="line">#       chk_apiserver</span><br><span class="line">#    &#125;</span><br><span class="line">&#125;</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>
</li>
<li><p>注意上述的健康检查是关闭的，集群建立完成后再开启：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">#    track_script &#123;</span><br><span class="line">#       chk_apiserver</span><br><span class="line">#    &#125;</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>配置KeepAlived健康检查文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt; &#x2F;etc&#x2F;keepalived&#x2F;check_apiserver.sh </span><br><span class="line">#!&#x2F;bin&#x2F;bash</span><br><span class="line"></span><br><span class="line">err&#x3D;0</span><br><span class="line">for k in $(seq 1 5)</span><br><span class="line">do</span><br><span class="line">    check_code&#x3D;$(pgrep kube-apiserver)</span><br><span class="line">    if [[ $check_code &#x3D;&#x3D; &quot;&quot; ]]; then</span><br><span class="line">        err&#x3D;$(expr $err + 1)</span><br><span class="line">        sleep 5</span><br><span class="line">        continue</span><br><span class="line">    else</span><br><span class="line">        err&#x3D;0</span><br><span class="line">        break</span><br><span class="line">    fi</span><br><span class="line">done</span><br><span class="line"></span><br><span class="line">if [[ $err !&#x3D; &quot;0&quot; ]]; then</span><br><span class="line">    echo &quot;systemctl stop keepalived&quot;</span><br><span class="line">    &#x2F;usr&#x2F;bin&#x2F;systemctl stop keepalived</span><br><span class="line">    exit 1</span><br><span class="line">else</span><br><span class="line">    exit 0</span><br><span class="line">fi</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>启动haproxy和keepalived</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">systemctl enable --now haproxy</span><br><span class="line">systemctl enable --now keepalived</span><br></pre></td></tr></table></figure>


</li>
</ul>
<h3 id="6-初始化集群"><a href="#6-初始化集群" class="headerlink" title="6. 初始化集群"></a>6. 初始化集群</h3><h4 id="6-1-官方安装文档："><a href="#6-1-官方安装文档：" class="headerlink" title="6.1 官方安装文档："></a>6.1 官方安装文档：</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https:&#x2F;&#x2F;kubernetes.io&#x2F;docs&#x2F;setup&#x2F;production-environment&#x2F;tools&#x2F;kubeadm&#x2F;high-availability&#x2F;</span><br></pre></td></tr></table></figure>



<h4 id="6-2-初始化"><a href="#6-2-初始化" class="headerlink" title="6.2 初始化"></a>6.2 初始化</h4><ul>
<li><p>各master节点配置<strong>kubeadm-config.yaml</strong>如下：</p>
<p>注意事项：</p>
<ol>
<li><strong>必须保证vip地址各节点能够正常访问</strong></li>
<li><strong>advertiseAddress为master01节点的ip</strong></li>
<li><strong>certSANs为vip的ip地址</strong></li>
<li><strong>controlPlaneEndpoint也是vip的ip地址</strong></li>
<li><strong>kubernetesVersion为当前kubernetes版本号</strong></li>
<li><strong>imageRepository备选地址：daocloud.io/daocloud</strong></li>
<li><strong>master01通过该配置文件初始化，master02和03通过改文件提前下载镜像</strong></li>
</ol>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt; &#x2F;root&#x2F;kubeadm-config.yaml</span><br><span class="line">apiVersion: kubeadm.k8s.io&#x2F;v1beta2</span><br><span class="line">bootstrapTokens:</span><br><span class="line">- groups:</span><br><span class="line">  - system:bootstrappers:kubeadm:default-node-token</span><br><span class="line">  token: 7t2weq.bjbawausm0jaxury</span><br><span class="line">  ttl: 24h0m0s</span><br><span class="line">  usages:</span><br><span class="line">  - signing</span><br><span class="line">  - authentication</span><br><span class="line">kind: InitConfiguration</span><br><span class="line">localAPIEndpoint:</span><br><span class="line">  advertiseAddress: 192.168.31.120</span><br><span class="line">  bindPort: 6443</span><br><span class="line">nodeRegistration:</span><br><span class="line">  criSocket: &#x2F;var&#x2F;run&#x2F;dockershim.sock</span><br><span class="line">  name: k8s-master01</span><br><span class="line">  taints:</span><br><span class="line">  - effect: NoSchedule</span><br><span class="line">    key: node-role.kubernetes.io&#x2F;master</span><br><span class="line">---</span><br><span class="line">apiServer:</span><br><span class="line">  certSANs:</span><br><span class="line">  - 192.168.31.125</span><br><span class="line">  timeoutForControlPlane: 4m0s</span><br><span class="line">apiVersion: kubeadm.k8s.io&#x2F;v1beta2</span><br><span class="line">certificatesDir: &#x2F;etc&#x2F;kubernetes&#x2F;pki</span><br><span class="line">clusterName: kubernetes</span><br><span class="line">controlPlaneEndpoint: 192.168.31.125:16443</span><br><span class="line">controllerManager: &#123;&#125;</span><br><span class="line">dns:</span><br><span class="line">  type: CoreDNS</span><br><span class="line">etcd:</span><br><span class="line">  local:</span><br><span class="line">    dataDir: &#x2F;var&#x2F;lib&#x2F;etcd</span><br><span class="line">imageRepository: registry.cn-hangzhou.aliyuncs.com&#x2F;google_containers</span><br><span class="line">kind: ClusterConfiguration</span><br><span class="line">kubernetesVersion: v1.19.3</span><br><span class="line">networking:</span><br><span class="line">  dnsDomain: cluster.local</span><br><span class="line">  podSubnet: 172.168.0.0&#x2F;16</span><br><span class="line">  serviceSubnet: 10.96.0.0&#x2F;12</span><br><span class="line">scheduler: &#123;&#125;</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>
</li>
<li><p>版本更新后配置文件可能存在过时的问题，通过以下命令更新即可</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm config migrate --old-config kubeadm-config.yaml --new-config new.yaml</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有Master节点提前下载镜像，可以节省初始化时间</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm config images pull --config &#x2F;root&#x2F;kubeadm-config.yaml</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有节点设置开机自启动kubelet</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">systemctl enable --now kubelet</span><br></pre></td></tr></table></figure>
</li>
<li><p>Master01节点初始化，初始化以后会在/etc/kubernetes目录下生成对应的证书和配置文件，之后其他Master节点加入Master01即可</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm init --config &#x2F;root&#x2F;kubeadm-config.yaml  --upload-certs</span><br></pre></td></tr></table></figure>

<p>另外可以不使用配置文件的方式初始化（仅供参考）</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm init --control-plane-endpoint &quot;LOAD_BALANCER_DNS:LOAD_BALANCER_PORT&quot; --upload-certs</span><br></pre></td></tr></table></figure>

<p>初始化失败时，可以使用以下命令重置</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubeadm reset</span><br></pre></td></tr></table></figure>
</li>
<li><p>初始化成功以后，会产生Token值，用于其他节点加入时使用，因此要记录下初始化成功生成的token值（令牌值）：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line">Your Kubernetes control-plane has initialized successfully!</span><br><span class="line"></span><br><span class="line">To start using your cluster, you need to run the following as a regular user:</span><br><span class="line"></span><br><span class="line">  mkdir -p $HOME&#x2F;.kube</span><br><span class="line">  sudo cp -i &#x2F;etc&#x2F;kubernetes&#x2F;admin.conf $HOME&#x2F;.kube&#x2F;config</span><br><span class="line">  sudo chown $(id -u):$(id -g) $HOME&#x2F;.kube&#x2F;config</span><br><span class="line"></span><br><span class="line">You should now deploy a pod network to the cluster.</span><br><span class="line">Run &quot;kubectl apply -f [podnetwork].yaml&quot; with one of the options listed at:</span><br><span class="line">  https:&#x2F;&#x2F;kubernetes.io&#x2F;docs&#x2F;concepts&#x2F;cluster-administration&#x2F;addons&#x2F;</span><br><span class="line"></span><br><span class="line">You can now join any number of the control-plane node running the following command on each as root:</span><br><span class="line"></span><br><span class="line">  kubeadm join 192.168.31.125:16443 --token 7t2weq.bjbawausm0jaxury \</span><br><span class="line">    --discovery-token-ca-cert-hash sha256:daa417fb1c31ad3dd378e67279ebc2c4b78fd60f33d0a6fc4c3911ee52ba17d2 \</span><br><span class="line">    --control-plane --certificate-key 4e0c01121d5a8dc7602f18e4af989ceef1a24b296bf7fe0cf0fa403fcefbacce</span><br><span class="line"></span><br><span class="line">Please note that the certificate-key gives access to cluster sensitive data, keep it secret!</span><br><span class="line">As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use</span><br><span class="line">&quot;kubeadm init phase upload-certs --upload-certs&quot; to reload certs afterward.</span><br><span class="line"></span><br><span class="line">Then you can join any number of worker nodes by running the following on each as root:</span><br><span class="line"></span><br><span class="line">kubeadm join 192.168.31.125:16443 --token 7t2weq.bjbawausm0jaxury \</span><br><span class="line">    --discovery-token-ca-cert-hash sha256:daa417fb1c31ad3dd378e67279ebc2c4b78fd60f33d0a6fc4c3911ee52ba17d2 </span><br><span class="line"></span><br></pre></td></tr></table></figure>
</li>
<li><p>查看集群状态命令</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po -n kube-system</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>配置集群管理员文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">mkdir -p $HOME&#x2F;.kube</span><br><span class="line">sudo cp -i &#x2F;etc&#x2F;kubernetes&#x2F;admin.conf $HOME&#x2F;.kube&#x2F;config</span><br><span class="line">sudo chown $(id -u):$(id -g) $HOME&#x2F;.kube&#x2F;config</span><br></pre></td></tr></table></figure>
</li>
<li><p>所有Master节点配置环境变量，用于访问Kubernetes集群</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF &gt;&gt; &#x2F;root&#x2F;.bashrc</span><br><span class="line">export KUBECONFIG&#x3D;&#x2F;etc&#x2F;kubernetes&#x2F;admin.conf</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">source &#x2F;root&#x2F;.bashrc</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="7-安装Calico"><a href="#7-安装Calico" class="headerlink" title="7. 安装Calico"></a>7. 安装Calico</h3><h4 id="7-1-Calico官网"><a href="#7-1-Calico官网" class="headerlink" title="7.1 Calico官网"></a>7.1 Calico官网</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https:&#x2F;&#x2F;www.projectcalico.org&#x2F;</span><br></pre></td></tr></table></figure>

<h4 id="7-2-Calico官方文档"><a href="#7-2-Calico官方文档" class="headerlink" title="7.2 Calico官方文档"></a>7.2 Calico官方文档</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https:&#x2F;&#x2F;docs.projectcalico.org&#x2F;getting-started&#x2F;kubernetes&#x2F;self-managed-onprem&#x2F;onpremises</span><br></pre></td></tr></table></figure>

<h4 id="7-3-安装步骤"><a href="#7-3-安装步骤" class="headerlink" title="7.3 安装步骤"></a>7.3 安装步骤</h4><ul>
<li><p>下载Calico的yaml文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl https:&#x2F;&#x2F;docs.projectcalico.org&#x2F;manifests&#x2F;calico.yaml -O</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>修改calico.yaml文件</p>
<ol>
<li>取消下面这两行的注释</li>
<li>将value的值设置为kubeadm-config.yaml文件中podSubnet一样的网段</li>
</ol>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">- name: CALICO_IPV4POOL_CIDR</span><br><span class="line">  value: &quot;172.168.0.0&#x2F;16&quot;</span><br></pre></td></tr></table></figure>
</li>
<li><p>初始化Caliaco</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f calico.yaml</span><br></pre></td></tr></table></figure>
</li>
<li><p>查看node状态</p>
<p>Caliaco镜像拉去完成后，node状态为Ready</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get nodes</span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="8-高可用master节点"><a href="#8-高可用master节点" class="headerlink" title="8. 高可用master节点"></a>8. 高可用master节点</h3><h4 id="8-1-初始化其他master节点加入集群"><a href="#8-1-初始化其他master节点加入集群" class="headerlink" title="8.1 初始化其他master节点加入集群"></a>8.1 初始化其他master节点加入集群</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubeadm join 192.168.31.125:16443 --token 7t2weq.bjbawausm0jaxury \</span><br><span class="line">  --discovery-token-ca-cert-hash sha256:daa417fb1c31ad3dd378e67279ebc2c4b78fd60f33d0a6fc4c3911ee52ba17d2 \</span><br><span class="line">  --control-plane --certificate-key 4e0c01121d5a8dc7602f18e4af989ceef1a24b296bf7fe0cf0fa403fcefbacce</span><br></pre></td></tr></table></figure>

<h4 id="8-2-master节点token过期后的处理方法"><a href="#8-2-master节点token过期后的处理方法" class="headerlink" title="8.2 master节点token过期后的处理方法"></a>8.2 master节点token过期后的处理方法</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"># Token过期后生成新的token：</span><br><span class="line">kubeadm token create --print-join-command</span><br><span class="line"></span><br><span class="line"># Master需要生成--certificate-key</span><br><span class="line">kubeadm init phase upload-certs  --upload-certs</span><br></pre></td></tr></table></figure>



<h3 id="9-node节点加入集群"><a href="#9-node节点加入集群" class="headerlink" title="9. node节点加入集群"></a>9. node节点加入集群</h3><h4 id="9-1-添加node节点"><a href="#9-1-添加node节点" class="headerlink" title="9.1 添加node节点"></a>9.1 添加node节点</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">kubeadm join 192.168.31.125:16443 --token 7t2weq.bjbawausm0jaxury \</span><br><span class="line">    --discovery-token-ca-cert-hash sha256:daa417fb1c31ad3dd378e67279ebc2c4b78fd60f33d0a6fc4c3911ee52ba17d2 </span><br></pre></td></tr></table></figure>

<h4 id="9-2-node节点token过期处理"><a href="#9-2-node节点token过期处理" class="headerlink" title="9.2 node节点token过期处理"></a>9.2 node节点token过期处理</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"># Token过期后生成新的token：</span><br><span class="line">kubeadm token create --print-join-command</span><br></pre></td></tr></table></figure>



<h3 id="10-Metrics-部署"><a href="#10-Metrics-部署" class="headerlink" title="10. Metrics 部署"></a>10. Metrics 部署</h3><p>在新版的Kubernetes中系统资源的采集均使用Metrics-server，可以通过Metrics采集节点和Pod的内存、磁盘、CPU和网络的使用率</p>
<h4 id="10-1-官方GitHub："><a href="#10-1-官方GitHub：" class="headerlink" title="10.1 官方GitHub："></a>10.1 官方GitHub：</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https:&#x2F;&#x2F;github.com&#x2F;kubernetes-sigs&#x2F;metrics-server</span><br></pre></td></tr></table></figure>

<h4 id="10-2-部署"><a href="#10-2-部署" class="headerlink" title="10.2 部署"></a>10.2 部署</h4><ul>
<li><p>修改k8s-ha-install中metrics的yaml文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">vim &#x2F;root&#x2F;k8s-ha-install&#x2F;metrics-server-3.6.1&#x2F;metrics-server-deployment.yaml</span><br></pre></td></tr></table></figure>

<p>内容如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">front-proxy-ca.pem  改为  front-proxy-ca.crt</span><br></pre></td></tr></table></figure>
</li>
<li><p>将Master01节点的front-proxy-ca.crt复制到所有Node节点</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">scp &#x2F;etc&#x2F;kubernetes&#x2F;pki&#x2F;front-proxy-ca.crt k8s-node01:&#x2F;etc&#x2F;kubernetes&#x2F;pki&#x2F;front-proxy-ca.crt;\</span><br><span class="line">scp &#x2F;etc&#x2F;kubernetes&#x2F;pki&#x2F;front-proxy-ca.crt k8s-node02:&#x2F;etc&#x2F;kubernetes&#x2F;pki&#x2F;front-proxy-ca.crt</span><br></pre></td></tr></table></figure>
</li>
<li><p>安装metrics</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl  create -f  metrics-server-3.6.1&#x2F;</span><br></pre></td></tr></table></figure>

<p>也可以部署最新版的metrics-server（也要修改证书的名字）</p>
</li>
</ul>
<h3 id="11-dashboard部署"><a href="#11-dashboard部署" class="headerlink" title="11. dashboard部署"></a>11. dashboard部署</h3><h4 id="11-1-官方GitHub："><a href="#11-1-官方GitHub：" class="headerlink" title="11.1 官方GitHub："></a>11.1 官方GitHub：</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https:&#x2F;&#x2F;github.com&#x2F;kubernetes&#x2F;dashboard</span><br></pre></td></tr></table></figure>

<h4 id="11-2-部署"><a href="#11-2-部署" class="headerlink" title="11.2 部署"></a>11.2 部署</h4><ul>
<li>根据官方github安装命令</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f https:&#x2F;&#x2F;raw.githubusercontent.com&#x2F;kubernetes&#x2F;dashboard&#x2F;v2.0.4&#x2F;aio&#x2F;deploy&#x2F;recommended.yaml</span><br></pre></td></tr></table></figure>

<h4 id="11-3-访问dashboard"><a href="#11-3-访问dashboard" class="headerlink" title="11.3 访问dashboard"></a>11.3 访问dashboard</h4><ul>
<li><p>编辑svc</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard</span><br></pre></td></tr></table></figure>

<p>更改dashboard的svc中type的值为NodePort</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201111210545426.png" alt="image-20201111210545426"></p>
</li>
<li><p>查看暴露的端口号</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get svc kubernetes-dashboard -n kubernetes-dashboard</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201110225709387.png" alt="image-20201110225709387"></p>
</li>
<li><p>创建管理员用户</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">cat &lt;&lt;EOF&gt;&gt; admin.yaml</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ServiceAccount</span><br><span class="line">metadata:</span><br><span class="line">  name: admin-user</span><br><span class="line">  namespace: kube-system</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: ClusterRoleBinding </span><br><span class="line">metadata: </span><br><span class="line">  name: admin-user</span><br><span class="line">  annotations:</span><br><span class="line">    rbac.authorization.kubernetes.io&#x2F;autoupdate: &quot;true&quot;</span><br><span class="line">roleRef:</span><br><span class="line">  apiGroup: rbac.authorization.k8s.io</span><br><span class="line">  kind: ClusterRole</span><br><span class="line">  name: cluster-admin</span><br><span class="line">subjects:</span><br><span class="line">- kind: ServiceAccount</span><br><span class="line">  name: admin-user</span><br><span class="line">  namespace: kube-system</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f admin.yaml -n kube-system</span><br></pre></td></tr></table></figure>
</li>
<li><p>查看token值</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk &#39;&#123;print $1&#125;&#39;)</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>游览器访问dashboard（使用htttps:://任意节点ip:端口号 访问 , 并使用token值登录）</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201110230224919.png" alt="image-20201110230224919"><strong>注意：需要使用https协议访问</strong></p>
</li>
</ul>
<ul>
<li><p>修改Kube-proxy为ipvs模式</p>
<p>进入编辑模式</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl edit cm kube-proxy -n kube-system</span><br></pre></td></tr></table></figure>

<p>修改为</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">mode: “ipvs”</span><br></pre></td></tr></table></figure>
</li>
<li><p>更新Kube-Proxy的Pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl patch daemonset kube-proxy -p &quot;&#123;\&quot;spec\&quot;:&#123;\&quot;template\&quot;:&#123;\&quot;metadata\&quot;:&#123;\&quot;annotations\&quot;:&#123;\&quot;date\&quot;:\&quot;&#96;date +&#39;%s&#39;&#96;\&quot;&#125;&#125;&#125;&#125;&#125;&quot; -n kube-system</span><br></pre></td></tr></table></figure>
</li>
<li><p>验证Kube-Proxy模式</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl 127.0.0.1:10249&#x2F;proxyMode</span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="12-验证集群"><a href="#12-验证集群" class="headerlink" title="12. 验证集群"></a>12. 验证集群</h3><h4 id="12-1-查看所有namespaces"><a href="#12-1-查看所有namespaces" class="headerlink" title="12.1 查看所有namespaces"></a>12.1 查看所有namespaces</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po --all-namespaces</span><br></pre></td></tr></table></figure>

<h4 id="12-2-查看pod监控数据"><a href="#12-2-查看pod监控数据" class="headerlink" title="12.2 查看pod监控数据"></a>12.2 查看pod监控数据</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl top po --all-namespaces</span><br></pre></td></tr></table></figure>

<h4 id="12-3-测试所有节点和service是否连通"><a href="#12-3-测试所有节点和service是否连通" class="headerlink" title="12.3 测试所有节点和service是否连通"></a>12.3 测试所有节点和service是否连通</h4><ul>
<li><p>获取service ip</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubectl get svc</span><br><span class="line"></span><br><span class="line">kubectl get svc -n kube-system</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201111205819374.png" alt="image-20201111205819374"></p>
</li>
<li><p>使用telnet ip port 测试是否能连通</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">telnet 10.96.0.1 443</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">telnet 10.96.1.10 53</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h4 id="12-4-pod和pod之间是否连通"><a href="#12-4-pod和pod之间是否连通" class="headerlink" title="12.4 pod和pod之间是否连通"></a>12.4 pod和pod之间是否连通</h4><ul>
<li><p>获取ip地址</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po --all-namespaces -owide</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201111210054498.png" alt="image-20201111210054498"></p>
</li>
<li><p>进入指定pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl exec -it calico-node-b82w2 -n kube-system -- sh</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ping  172.168.32.131</span><br></pre></td></tr></table></figure>

</li>
</ul>
<h4 id=""><a href="#" class="headerlink" title=""></a></h4><h3 id="13-kuboard-另一种dashboard"><a href="#13-kuboard-另一种dashboard" class="headerlink" title="13. kuboard:  另一种dashboard"></a>13. kuboard:  另一种dashboard</h3><p>因为dashboard在使用时不是很方便，所以这里推荐一个国内开源的dashboard替代品：kuboard</p>
<ul>
<li><p>官网</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">www.kuboard.cn</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>安装</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f https:&#x2F;&#x2F;addons.kuboard.cn&#x2F;metrics-server&#x2F;0.3.7&#x2F;metrics-server.yaml</span><br></pre></td></tr></table></figure>
</li>
<li><p>获取token</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">echo $(kubectl -n kube-system get secret $(kubectl -n kube-system get secret | grep kuboard-user | awk &#39;&#123;print $1&#125;&#39;) -o go-template&#x3D;&#39;&#123;&#123;.data.token&#125;&#125;&#39; | base64 -d)</span><br></pre></td></tr></table></figure>
</li>
<li><p>查看svc</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get svc -n kube-system</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201111211354870.png" alt="image-20201111211354870"></p>
</li>
</ul>
<ul>
<li><p>游览器访问ip:端口，使用获取的token登录即可</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201111211516884.png" alt="image-20201111211516884"></p>
</li>
</ul>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">k8s安装</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/kubeadm%E5%AE%89%E8%A3%85k8s%E9%AB%98%E5%8F%AF%E7%94%A8%E9%9B%86%E7%BE%A4/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s基础ConfigMap 和 Secret" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80ConfigMap%20%E5%92%8C%20Secret/">3.k8s基础ConfigMap和Secret</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80ConfigMap%20%E5%92%8C%20Secret/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-ConfigMap"><a href="#1-ConfigMap" class="headerlink" title="1 ConfigMap"></a>1 ConfigMap</h1><h2 id="1-1-什么是configmap"><a href="#1-1-什么是configmap" class="headerlink" title="1.1 什么是configmap"></a>1.1 什么是configmap</h2><p>ConfigMap 是一种 API 对象，用来将非机密性的数据保存到键值对中。使用时， <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/">Pods</a> 可以将其用作环境变量、命令行参数或者存储卷中的配置文件。</p>
<p>ConfigMap 将您的环境配置信息和 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/glossary/?all=true#term-image">容器镜像</a> 解耦，便于应用配置的修改。</p>
<blockquote>
<p><strong>注意：</strong></p>
<p>ConfigMap 并不提供保密或者加密功能。 如果你想存储的数据是机密的，请使用 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/configuration/secret/">Secret</a>， 或者使用其他第三方工具来保证你的数据的私密性，而不是用 ConfigMap。</p>
</blockquote>
<h2 id="1-2-configmap应用场景"><a href="#1-2-configmap应用场景" class="headerlink" title="1.2 configmap应用场景"></a>1.2 configmap应用场景</h2><p>使用 ConfigMap 来将你的配置数据和应用程序代码分开。</p>
<p>比如，假设你正在开发一个应用，它可以在你自己的电脑上（用于开发）和在云上 （用于实际流量）运行。 你的代码里有一段是用于查看环境变量 <code>DATABASE_HOST</code>，在本地运行时， 你将这个变量设置为 <code>localhost</code>，在云上，可以设置为在 Kubernetes 集群中的service。</p>
<p>ConfigMap 在设计上不是用来保存大量数据的。在 ConfigMap 中保存的数据不可超过 1 MiB。如果你需要保存超出此尺寸限制的数据，你可能希望考虑挂载存储卷 或者使用独立的数据库或者文件服务。</p>
<h2 id="1-3-创建-ConfigMap"><a href="#1-3-创建-ConfigMap" class="headerlink" title="1.3 创建 ConfigMap"></a>1.3 创建 ConfigMap</h2><p>你可以使用 <code>kubectl create configmap</code> 或者在 <code>kustomization.yaml</code> 中的 ConfigMap 生成器 来创建 ConfigMap。</p>
<blockquote>
<p>注意，<code>kubectl</code> 从 1.14 版本开始支持 <code>kustomization.yaml</code></p>
</blockquote>
<h3 id="1-3-1-使用-kubectl-create-configmap-创建-ConfigMap"><a href="#1-3-1-使用-kubectl-create-configmap-创建-ConfigMap" class="headerlink" title="1.3.1 使用 kubectl create configmap 创建 ConfigMap"></a>1.3.1 使用 kubectl create configmap 创建 ConfigMap</h3><p>你可以使用 <code>kubectl create configmap</code> 命令基于 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-pod-configmap/#create-configmaps-from-directories">目录</a>、<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-pod-configmap/#create-configmaps-from-files">文件</a> 或者<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-pod-configmap/#create-configmaps-from-literal-values">字面值</a>来创建 ConfigMap：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap &lt;map-name&gt; &lt;data-source&gt;</span><br></pre></td></tr></table></figure>

<p>其中，<map-name> 是要设置的 ConfigMap 名称，<data-source> 是要从中提取数据的目录、 文件或者字面值。 ConfigMap 对象的名称必须是合法的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/names#dns-subdomain-names">DNS 子域名</a>.</p>
<p>在你基于文件来创建 ConfigMap 时，<data-source> 中的键名默认取自 文件的名称，而对应的值则默认为文件的内容。</p>
<p>你可以使用<a target="_blank" rel="noopener" href="https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands/#describe"><code>kubectl describe</code></a> 或者 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands/#get"><code>kubectl get</code></a> 获取有关 ConfigMap 的信息。</p>
<h3 id="1-3-2-基于目录创建-ConfigMap"><a href="#1-3-2-基于目录创建-ConfigMap" class="headerlink" title="1.3.2 基于目录创建 ConfigMap"></a>1.3.2 基于目录创建 ConfigMap</h3><p>你可以使用 <code>kubectl create configmap</code> 基于同一目录中的多个文件创建 ConfigMap。 当你基于目录来创建 ConfigMap 时，kubectl 识别目录下基本名可以作为合法键名的 文件，并将这些文件打包到新的 ConfigMap 中。普通文件之外的所有目录项都会被 忽略（例如，子目录、符号链接、设备、管道等等）。</p>
<p>例如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建本地目录</span></span><br><span class="line">mkdir -p configure-pod-container/configmap/</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 创建文件game.properties </span></span><br><span class="line">cat &lt;&lt;EOF&gt; configure-pod-container/configmap/game.properties </span><br><span class="line">enemies=aliens</span><br><span class="line">lives=3</span><br><span class="line">enemies.cheat=true</span><br><span class="line">enemies.cheat.level=noGoodRotten</span><br><span class="line">secret.code.passphrase=UUDDLRLRBABAS</span><br><span class="line">secret.code.allowed=true</span><br><span class="line">secret.code.lives=30</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 创建文件ui.properties </span></span><br><span class="line">cat &lt;&lt;EOF&gt; configure-pod-container/configmap/ui.properties </span><br><span class="line">color.good=purple</span><br><span class="line">color.bad=yellow</span><br><span class="line">allow.textmode=true</span><br><span class="line">how.nice.to.look=fairlyNic</span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 创建 configmap</span></span><br><span class="line">kubectl create configmap game-config --from-file=configure-pod-container/configmap/</span><br></pre></td></tr></table></figure>

<p>以上命令将 <code>configure-pod-container/configmap</code> 目录下的所有文件，也就是 <code>game.properties</code> 和 <code>ui.properties</code> 打包到 game-config ConfigMap 中。你可以使用下面的命令显示 ConfigMap 的详细信息：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl describe configmaps game-config</span><br></pre></td></tr></table></figure>

<p>输出类似以下内容：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">Name:         game-config</span><br><span class="line">Namespace:    default</span><br><span class="line">Labels:       &lt;none&gt;</span><br><span class="line">Annotations:  &lt;none&gt;</span><br><span class="line"></span><br><span class="line">Data</span><br><span class="line">&#x3D;&#x3D;&#x3D;&#x3D;</span><br><span class="line">game.properties:</span><br><span class="line">----</span><br><span class="line">enemies&#x3D;aliens</span><br><span class="line">lives&#x3D;3</span><br><span class="line">enemies.cheat&#x3D;true</span><br><span class="line">enemies.cheat.level&#x3D;noGoodRotten</span><br><span class="line">secret.code.passphrase&#x3D;UUDDLRLRBABAS</span><br><span class="line">secret.code.allowed&#x3D;true</span><br><span class="line">secret.code.lives&#x3D;30</span><br><span class="line">ui.properties:</span><br><span class="line">----</span><br><span class="line">color.good&#x3D;purple</span><br><span class="line">color.bad&#x3D;yellow</span><br><span class="line">allow.textmode&#x3D;true</span><br><span class="line">how.nice.to.look&#x3D;fairlyNice</span><br></pre></td></tr></table></figure>

<p><code>configure-pod-container/configmap/</code> 目录中的 <code>game.properties</code> 和 <code>ui.properties</code> 文件出现在 ConfigMap 的 <code>data</code> 部分。</p>
<h3 id="1-3-3-基于文件创建-ConfigMap"><a href="#1-3-3-基于文件创建-ConfigMap" class="headerlink" title="1.3.3 基于文件创建 ConfigMap"></a>1.3.3 基于文件创建 ConfigMap</h3><p>可以使用 <code>kubectl create configmap</code> 基于单个文件或多个文件创建 ConfigMap。</p>
<p>例如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap game-config-2 --from-file=configure-pod-container/configmap/game.properties</span><br></pre></td></tr></table></figure>



<p>可以多次使用 <code>--from-file</code> 参数，从多个数据源创建 ConfigMap。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap game-config-2 --from-file&#x3D;configure-pod-container&#x2F;configmap&#x2F;game.properties --from-file&#x3D;configure-pod-container&#x2F;configmap&#x2F;ui.properties</span><br></pre></td></tr></table></figure>



<h3 id="1-3-4-从env环境文件创建-ConfigMap"><a href="#1-3-4-从env环境文件创建-ConfigMap" class="headerlink" title="1.3.4 从env环境文件创建 ConfigMap"></a>1.3.4 从env环境文件创建 ConfigMap</h3><p>使用 <code>--from-env-file</code> 选项从环境文件创建 ConfigMap，例如：</p>
<p>Env 文件包含环境变量列表。 其中适用以下语法规则:</p>
<ul>
<li>Env 文件中的每一行必须为 VAR=VAL 格式。</li>
<li>以＃开头的行（即注释）将被忽略。</li>
<li>空行将被忽略。</li>
<li>引号不会被特殊处理（即它们将成为 ConfigMap 值的一部分）。</li>
<li><strong>注意：</strong> 当多次使用 <code>--from-env-file</code> 从多个数据源创建 ConfigMap 时，仅仅最后一个 env 文件有效。</li>
</ul>
<p>例如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建文件game-env-file.properties</span></span><br><span class="line">cat &lt;&lt;EOF&gt;game-env-file.properties</span><br><span class="line">enemies=aliens</span><br><span class="line">lives=3</span><br><span class="line">allowed=&quot;true&quot;</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> This comment and the empty line above it are ignored</span></span><br><span class="line"></span><br><span class="line">EOF</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">使用 `--from-env-file` 选项从环境文件创建 ConfigMap</span></span><br><span class="line">kubectl create configmap game-config-env-file \</span><br><span class="line">       --from-env-file=./game-env-file.properties</span><br></pre></td></tr></table></figure>



<h3 id="1-3-5-定义从文件创建-ConfigMap-时要使用的键"><a href="#1-3-5-定义从文件创建-ConfigMap-时要使用的键" class="headerlink" title="1.3.5 定义从文件创建 ConfigMap 时要使用的键"></a>1.3.5 定义从文件创建 ConfigMap 时要使用的键</h3><p>在使用 <code>--from-file</code> 参数时，你可以定义在 ConfigMap 的 <code>data</code> 部分出现键名， 而不是按默认行为使用文件名：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap game-config-3 --from-file=&lt;my-key-name&gt;=&lt;path-to-file&gt;</span><br></pre></td></tr></table></figure>

<p><code>&lt;my-key-name&gt;</code> 是你要在 ConfigMap 中使用的键名，<code>&lt;path-to-file&gt;</code> 表示数据源文件的位置。</p>
<p>例如:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap game-config-3 --from-file=game-special-key=configure-pod-container/configmap/game.properties</span><br></pre></td></tr></table></figure>

<p>将产生以下 ConfigMap:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get configmaps game-config-3 -o yaml</span><br></pre></td></tr></table></figure>

<p>输出类似以下内容：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">creationTimestamp:</span> <span class="number">2016-02-18T18:54:22Z</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">game-config-3</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">resourceVersion:</span> <span class="string">&quot;530&quot;</span></span><br><span class="line">  <span class="attr">selfLink:</span> <span class="string">/api/v1/namespaces/default/configmaps/game-config-3</span></span><br><span class="line">  <span class="attr">uid:</span> <span class="string">05f8da22-d671-11e5-8cd0-68f728db1985</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">game-special-key:</span> <span class="string">|</span></span><br><span class="line">    <span class="string">enemies=aliens</span></span><br><span class="line">    <span class="string">lives=3</span></span><br><span class="line">    <span class="string">enemies.cheat=true</span></span><br><span class="line">    <span class="string">enemies.cheat.level=noGoodRotten</span></span><br><span class="line">    <span class="string">secret.code.passphrase=UUDDLRLRBABAS</span></span><br><span class="line">    <span class="string">secret.code.allowed=true</span></span><br><span class="line">    <span class="string">secret.code.lives=30</span>    </span><br></pre></td></tr></table></figure>



<h3 id="1-3-6-根据字面值创建-ConfigMap"><a href="#1-3-6-根据字面值创建-ConfigMap" class="headerlink" title="1.3.6 根据字面值创建 ConfigMap"></a>1.3.6 根据字面值创建 ConfigMap</h3><p>你可以将 <code>kubectl create configmap</code> 与 <code>--from-literal</code> 参数一起使用，从命令行定义文字值:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap special-config --from-literal=special.how=very --from-literal=special.type=charm</span><br></pre></td></tr></table></figure>

<p>你可以传入多个键值对。命令行中提供的每对键值在 ConfigMap 的 <code>data</code> 部分中均表示为单独的条目。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get configmaps special-config -o yaml</span><br></pre></td></tr></table></figure>

<p>输出类似以下内容:</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">creationTimestamp:</span> <span class="number">2016-02-18T19:14:38Z</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">resourceVersion:</span> <span class="string">&quot;651&quot;</span></span><br><span class="line">  <span class="attr">selfLink:</span> <span class="string">/api/v1/namespaces/default/configmaps/special-config</span></span><br><span class="line">  <span class="attr">uid:</span> <span class="string">dadce046-d673-11e5-8cd0-68f728db1985</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">special.how:</span> <span class="string">very</span></span><br><span class="line">  <span class="attr">special.type:</span> <span class="string">charm</span></span><br></pre></td></tr></table></figure>





<h3 id="1-3-7-基于生成器创建-ConfigMap"><a href="#1-3-7-基于生成器创建-ConfigMap" class="headerlink" title="1.3.7 基于生成器创建 ConfigMap"></a>1.3.7 基于生成器创建 ConfigMap</h3><p>自 1.14 开始，<code>kubectl</code> 开始支持 <code>kustomization.yaml</code>。 你还可以基于生成器创建 ConfigMap，然后将其应用于 API 服务器上创建对象。 生成器应在目录内的 <code>kustomization.yaml</code> 中指定。</p>
<h4 id="1-3-7-1-基于文件生成-ConfigMap"><a href="#1-3-7-1-基于文件生成-ConfigMap" class="headerlink" title="1.3.7.1 基于文件生成 ConfigMap"></a>1.3.7.1 基于文件生成 ConfigMap</h4><p>例如，要从 <code>configure-pod-container/configmap/kubectl/game.properties</code> 文件生成一个 ConfigMap：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建包含 ConfigMapGenerator 的 kustomization.yaml 文件</span></span><br><span class="line">cat &lt;&lt;EOF &gt;./kustomization.yaml</span><br><span class="line">configMapGenerator:</span><br><span class="line">- name: game-config-4</span><br><span class="line">  files:</span><br><span class="line">  - configure-pod-container/configmap/kubectl/game.properties</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>

<p>使用 kustomization 目录创建 ConfigMap 对象：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -k .</span><br></pre></td></tr></table></figure>

<blockquote>
<p>生成的 ConfigMap 名称是通过对内容进行hash而附加的后缀， 这样可以确保每次修改内容时都会生成新的 ConfigMap。</p>
</blockquote>
<h4 id="1-3-7-2-定义从文件生成-ConfigMap-时要使用的键"><a href="#1-3-7-2-定义从文件生成-ConfigMap-时要使用的键" class="headerlink" title="1.3.7.2 定义从文件生成 ConfigMap 时要使用的键"></a>1.3.7.2 定义从文件生成 ConfigMap 时要使用的键</h4><p>在 ConfigMap 生成器，你可以定义一个非文件名的键名。 例如，从 <code>configure-pod-container/configmap/game.properties</code> 文件生成 ConfigMap， 但使用 <code>game-special-key</code> 作为键名：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建包含 ConfigMapGenerator 的 kustomization.yaml 文件</span></span><br><span class="line">cat &lt;&lt;EOF &gt;./kustomization.yaml</span><br><span class="line">configMapGenerator:</span><br><span class="line">- name: game-config-5</span><br><span class="line">  files:</span><br><span class="line">  - game-special-key=configure-pod-container/configmap/kubectl/game.properties</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>

<p>使用 Kustomization 目录创建 ConfigMap 对象。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -k .</span><br></pre></td></tr></table></figure>



<h4 id="1-3-7-3-从字面值生成-ConfigMap"><a href="#1-3-7-3-从字面值生成-ConfigMap" class="headerlink" title="1.3.7.3 从字面值生成 ConfigMap"></a>1.3.7.3 从字面值生成 ConfigMap</h4><p>要基于字符串 <code>special.type=charm</code> 和 <code>special.how=very</code> 生成 ConfigMap， 可以在 <code>kusotmization.yaml</code> 中配置 ConfigMap 生成器：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建带有 ConfigMapGenerator 的 kustomization.yaml 文件</span></span><br><span class="line">cat &lt;&lt;EOF &gt;./kustomization.yaml</span><br><span class="line">configMapGenerator:</span><br><span class="line">- name: special-config-2</span><br><span class="line">  literals:</span><br><span class="line">  - special.how=very</span><br><span class="line">  - special.type=charm</span><br><span class="line">EOF</span><br></pre></td></tr></table></figure>

<p>应用 Kustomization 目录创建 ConfigMap 对象。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -k .</span><br></pre></td></tr></table></figure>







<h2 id="1-4-使用ConfigMap"><a href="#1-4-使用ConfigMap" class="headerlink" title="1.4 使用ConfigMap"></a>1.4 使用ConfigMap</h2><h3 id="1-4-1-使用-ConfigMap-数据定义容器环境变量"><a href="#1-4-1-使用-ConfigMap-数据定义容器环境变量" class="headerlink" title="1.4.1 使用 ConfigMap 数据定义容器环境变量"></a>1.4.1 使用 ConfigMap 数据定义容器环境变量</h3><h4 id="1-4-1-1使用单个-ConfigMap-中的数据定义容器环境变量"><a href="#1-4-1-1使用单个-ConfigMap-中的数据定义容器环境变量" class="headerlink" title="1.4.1.1使用单个 ConfigMap 中的数据定义容器环境变量"></a>1.4.1.1使用单个 ConfigMap 中的数据定义容器环境变量</h4><ol>
<li><p>在 ConfigMap 中将环境变量定义为键值对:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create configmap special-config --from-literal=special.how=very</span><br></pre></td></tr></table></figure>
</li>
<li><p>将 ConfigMap 中定义的 <code>special.how</code> 值分配给 Pod 规范中的 <code>SPECIAL_LEVEL_KEY</code> 环境变量。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>, <span class="string">&quot;-c&quot;</span>, <span class="string">&quot;env&quot;</span> ]</span><br><span class="line">      <span class="attr">env:</span></span><br><span class="line">        <span class="comment"># 定义一个环境变量</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SPECIAL_LEVEL_KEY</span></span><br><span class="line">          <span class="attr">valueFrom:</span></span><br><span class="line">            <span class="attr">configMapKeyRef:</span></span><br><span class="line">              <span class="comment"># 指定configmap的名称</span></span><br><span class="line">              <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">              <span class="comment"># 指定需要获取值的键名</span></span><br><span class="line">              <span class="attr">key:</span> <span class="string">special.how</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>创建 Pod:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-single-configmap-env-variable.yaml</span><br></pre></td></tr></table></figure>

<blockquote>
<p>创建 Pod后，Pod 的输出包含环境变量 <code>SPECIAL_LEVEL_KEY=very</code>。</p>
</blockquote>
</li>
</ol>
<h4 id="1-4-1-2-使用来自多个-ConfigMap-的数据定义容器环境变量"><a href="#1-4-1-2-使用来自多个-ConfigMap-的数据定义容器环境变量" class="headerlink" title="1.4.1.2 使用来自多个 ConfigMap 的数据定义容器环境变量"></a>1.4.1.2 使用来自多个 ConfigMap 的数据定义容器环境变量</h4><ul>
<li><p>与前面的示例一样，首先创建 ConfigMap。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">special.how:</span> <span class="string">very</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">env-config</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">log_level:</span> <span class="string">INFO</span></span><br></pre></td></tr></table></figure>

<p>创建 ConfigMap:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f configmaps.yaml</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>在 Pod 中定义环境变量。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>, <span class="string">&quot;-c&quot;</span>, <span class="string">&quot;env&quot;</span> ]</span><br><span class="line">      <span class="attr">env:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SPECIAL_LEVEL_KEY</span></span><br><span class="line">          <span class="attr">valueFrom:</span></span><br><span class="line">            <span class="attr">configMapKeyRef:</span></span><br><span class="line">              <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">              <span class="attr">key:</span> <span class="string">special.how</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">LOG_LEVEL</span></span><br><span class="line">          <span class="attr">valueFrom:</span></span><br><span class="line">            <span class="attr">configMapKeyRef:</span></span><br><span class="line">              <span class="attr">name:</span> <span class="string">env-config</span></span><br><span class="line">              <span class="attr">key:</span> <span class="string">log_level</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>创建 Pod:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-multiple-configmap-env-variable.yaml</span><br></pre></td></tr></table></figure>

<blockquote>
<p>创建 Pod,Pod 的输出包含环境变量 <code>SPECIAL_LEVEL_KEY=very</code> 和 <code>LOG_LEVEL=INFO</code>。</p>
</blockquote>
</li>
</ul>
<h4 id="1-4-1-3-将-ConfigMap-中的所有键值对自动配置为容器的环境变量"><a href="#1-4-1-3-将-ConfigMap-中的所有键值对自动配置为容器的环境变量" class="headerlink" title="1.4.1.3 将 ConfigMap 中的所有键值对自动配置为容器的环境变量"></a>1.4.1.3 将 ConfigMap 中的所有键值对自动配置为容器的环境变量</h4><p>此功能相比之前手动创建环境变量的方法更加简单，所以也是最常用到的功能</p>
<blockquote>
<p>说明：Kubernetes v1.6 和更高版本支持此功能。</p>
</blockquote>
<ul>
<li><p>创建一个包含多个键值对的 ConfigMap。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">SPECIAL_LEVEL:</span> <span class="string">very</span></span><br><span class="line">  <span class="attr">SPECIAL_TYPE:</span> <span class="string">charm</span></span><br></pre></td></tr></table></figure>

<p>创建 ConfigMap:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f configmap-multikeys.yaml</span><br></pre></td></tr></table></figure>
</li>
<li><p>使用 <code>envFrom</code> 将所有 ConfigMap 的数据定义为容器环境变量，ConfigMap 中的键成为 Pod 中的环境变量名称。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>, <span class="string">&quot;-c&quot;</span>, <span class="string">&quot;env&quot;</span> ]</span><br><span class="line">      <span class="attr">envFrom:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">configMapRef:</span></span><br><span class="line">          <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>创建 Pod:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-configmap-envFrom.yaml</span><br></pre></td></tr></table></figure>

<blockquote>
<p>Pod 的输出包含环境变量 <code>SPECIAL_LEVEL=very</code> 和 <code>SPECIAL_TYPE=charm</code>。</p>
</blockquote>
</li>
</ul>
<h3 id="1-4-2-在-Pod-命令中使用-ConfigMap-定义的环境变量"><a href="#1-4-2-在-Pod-命令中使用-ConfigMap-定义的环境变量" class="headerlink" title="1.4.2 在 Pod 命令中使用 ConfigMap 定义的环境变量"></a>1.4.2 在 Pod 命令中使用 ConfigMap 定义的环境变量</h3><p>你可以使用 <code>$(VAR_NAME)</code> Kubernetes 替换语法在容器的 <code>command</code> 和 <code>args</code> 部分中使用 ConfigMap 定义的环境变量。</p>
<p>例如，以下 Pod 规范:</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>, <span class="string">&quot;-c&quot;</span>, <span class="string">&quot;echo $(SPECIAL_LEVEL_KEY) $(SPECIAL_TYPE_KEY)&quot;</span> ]</span><br><span class="line">      <span class="attr">env:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SPECIAL_LEVEL_KEY</span></span><br><span class="line">          <span class="attr">valueFrom:</span></span><br><span class="line">            <span class="attr">configMapKeyRef:</span></span><br><span class="line">              <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">              <span class="attr">key:</span> <span class="string">SPECIAL_LEVEL</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SPECIAL_TYPE_KEY</span></span><br><span class="line">          <span class="attr">valueFrom:</span></span><br><span class="line">            <span class="attr">configMapKeyRef:</span></span><br><span class="line">              <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">              <span class="attr">key:</span> <span class="string">SPECIAL_TYPE</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>通过运行下面命令创建 Pod：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-configmap-env-var-valueFrom.yaml</span><br></pre></td></tr></table></figure>

<p>在 <code>test-container</code> 容器中产生以下输出:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">very charm</span><br></pre></td></tr></table></figure>



<h3 id="1-4-3-将-ConfigMap-数据添加到一个卷中"><a href="#1-4-3-将-ConfigMap-数据添加到一个卷中" class="headerlink" title="1.4.3 将 ConfigMap 数据添加到一个卷中"></a>1.4.3 将 ConfigMap 数据添加到一个卷中</h3><p>当你使用 <code>--from-file</code> 创建 ConfigMap 时，文件名成为存储在 ConfigMap 的 <code>data</code> 部分中的键， 文件内容成为键对应的值。</p>
<p>后面的示例中引用了一个名为 special-config 的 ConfigMap，如下所示：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">SPECIAL_LEVEL:</span> <span class="string">very</span></span><br><span class="line">  <span class="attr">SPECIAL_TYPE:</span> <span class="string">charm</span></span><br></pre></td></tr></table></figure>

<p>创建 ConfigMap:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f configmap-multikeys.yaml</span><br></pre></td></tr></table></figure>

<h4 id="1-4-3-1-使用存储在-ConfigMap-中的数据填充数据卷"><a href="#1-4-3-1-使用存储在-ConfigMap-中的数据填充数据卷" class="headerlink" title="1.4.3.1 使用存储在 ConfigMap 中的数据填充数据卷"></a>1.4.3.1 使用存储在 ConfigMap 中的数据填充数据卷</h4><p>在 Pod 的 <code>volumes</code> 部分下添加 ConfigMap 名称。 这会将 ConfigMap 数据添加到指定为 <code>volumeMounts.mountPath</code> 的目录中（在本例中为 <code>/etc/config</code>）。 <code>command</code> 部分引用存储在 ConfigMap 中的 <code>special.level</code>。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>, <span class="string">&quot;-c&quot;</span>, <span class="string">&quot;ls /etc/config/&quot;</span> ]</span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">        <span class="attr">mountPath:</span> <span class="string">/etc/config</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">      <span class="attr">configMap:</span></span><br><span class="line">        <span class="comment"># Provide the name of the ConfigMap containing the files you want</span></span><br><span class="line">        <span class="comment"># to add to the container</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>创建 Pod:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-configmap-volume.yaml</span><br></pre></td></tr></table></figure>

<p>Pod 运行时，命令 <code>ls /etc/config/</code> 产生下面的输出：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">SPECIAL_LEVEL</span><br><span class="line">SPECIAL_TYPE</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>注意：</strong> 如果在 <code>/etc/config/</code> 目录中存在文件，它们会被删除。</p>
</blockquote>
<blockquote>
<p><strong>说明：</strong> 文本数据默认使用 UTF-8 字符编码。如果使用其他字符编码， 可以使用 <code>binaryData</code>指定。</p>
</blockquote>
<h4 id="1-4-3-2-将-ConfigMap-数据添加到数据卷中的指定文件"><a href="#1-4-3-2-将-ConfigMap-数据添加到数据卷中的指定文件" class="headerlink" title="1.4.3.2 将 ConfigMap 数据添加到数据卷中的指定文件"></a>1.4.3.2 将 ConfigMap 数据添加到数据卷中的指定文件</h4><p>使用 <code>path</code> 字段为特定的 ConfigMap键指定文件路径 。 在这里，<code>SPECIAL_LEVEL</code> 将挂载在 <code>config-volume</code> 数据卷中 <code>/etc/config/keys</code> 文件中。</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">dapi-test-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">test-container</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">k8s.gcr.io/busybox</span></span><br><span class="line">      <span class="attr">command:</span> [ <span class="string">&quot;/bin/sh&quot;</span>,<span class="string">&quot;-c&quot;</span>,<span class="string">&quot;cat /etc/config/keys&quot;</span> ]</span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">        <span class="attr">mountPath:</span> <span class="string">/etc/config</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">      <span class="attr">configMap:</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">special-config</span></span><br><span class="line">        <span class="attr">items:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">key:</span> <span class="string">SPECIAL_LEVEL</span></span><br><span class="line">          <span class="attr">path:</span> <span class="string">keys</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<p>创建Pod：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod-configmap-volume-specific-key.yaml</span><br></pre></td></tr></table></figure>

<p>当 pod 运行时，命令 <code>cat /etc/config/keys</code> 产生以下输出：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">very</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>注意：</strong> 如前，<code>/etc/config/</code> 目录中所有先前的文件都被删除。</p>
</blockquote>
<h2 id="1-5-ConfigMap的特性"><a href="#1-5-ConfigMap的特性" class="headerlink" title="1.5 ConfigMap的特性"></a>1.5 ConfigMap的特性</h2><h3 id="1-5-1-挂载的-ConfigMap-将自动更新"><a href="#1-5-1-挂载的-ConfigMap-将自动更新" class="headerlink" title="1.5.1 挂载的 ConfigMap 将自动更新"></a>1.5.1 挂载的 ConfigMap 将自动更新</h3><p>更新已经在数据卷中使用的 ConfigMap 时，已映射的键最终也会被更新。 <code>kubelet</code> 在每次定期同步时都会检查已挂载的 ConfigMap 是否是最新的。 但是，它使用其本地的基于 TTL 的缓存来获取 ConfigMap 的当前值。 因此，从更新 ConfigMap 到将新键映射到 Pod 的总延迟可能与 kubelet 同步周期 + ConfigMap 在 kubelet 中缓存的 TTL 一样长。</p>
<blockquote>
<p><strong>说明：</strong> 使用 ConfigMap 作为 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/volumes/#using-subpath">subPath</a> 的数据卷将不会收到 ConfigMap 更新。</p>
</blockquote>
<h3 id="1-5-2-限制"><a href="#1-5-2-限制" class="headerlink" title="1.5.2 限制"></a>1.5.2 限制</h3><ul>
<li><p>在 Pod 中引用 ConfigMap之前，必须先创建ConfigMap（除非将 ConfigMap 标记为”可选”）。 如果引用的 ConfigMap 不存在，则 Pod 将不会启动。同样，引用 ConfigMap 中不存在的键也会使 Pod 启动失败。</p>
</li>
<li><p>如果你使用 <code>envFrom</code> 基于 ConfigMap 定义环境变量，那么无效的键将被忽略。 可以启动 Pod，但无效名称将记录在事件日志中（<code>InvalidVariableNames</code>）。 日志消息列出了每个被忽略的键。例如:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get events</span><br></pre></td></tr></table></figure>

<p>输出与此类似:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">LASTSEEN FIRSTSEEN COUNT NAME          KIND  SUBOBJECT  TYPE      REASON                            SOURCE                MESSAGE</span><br><span class="line">0s       0s        1     dapi-test-pod Pod              Warning   InvalidEnvironmentVariableNames   &#123;kubelet, 127.0.0.1&#125;  Keys [1badkey, 2alsobad] from the EnvFrom configMap default&#x2F;myconfig were skipped since they are considered invalid environment variable names.</span><br></pre></td></tr></table></figure>
</li>
<li><p>ConfigMap 位于特定的<code>namespace</code>中。ConfigMap 只能被同一<code>namespace</code>中的 Pod 引用.</p>
</li>
<li><p>你不能将 ConfigMap 用于 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configure-pod-container/static-pod/">静态 Pod</a>， 因为 Kubernetes 不支持这种用法。</p>
</li>
</ul>
<p>&lt;&lt;&lt;&lt;&lt;&lt;&lt; HEAD</p>
<h1 id="2-Secret"><a href="#2-Secret" class="headerlink" title="2 Secret"></a>2 Secret</h1><p><code>Secret</code> 对象类型用来保存敏感信息，例如密码、OAuth 令牌和 SSH 密钥。 将这些信息放在 <code>secret</code> 中比放在 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/">Pod</a> 的定义或者 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/glossary/?all=true#term-image">容器镜像</a> 中来说更加安全和灵活。</p>
<p>详细文档参考官方： <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/configuration/secret/">https://kubernetes.io/zh/docs/concepts/configuration/secret/</a></p>
<h2 id="2-1-Secret-的类型"><a href="#2-1-Secret-的类型" class="headerlink" title="2.1 Secret 的类型"></a>2.1 Secret 的类型</h2><p>在创建 Secret 对象时，你可以使用 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secret-v1-core"><code>Secret</code></a> 资源的 <code>type</code> 字段，或者与其等价的 <code>kubectl</code> 命令行参数（如果有的话）为其设置类型。 Secret 的类型用来帮助编写程序处理 Secret 数据。</p>
<p>Kubernetes 提供若干种内置的类型，用于一些常见的使用场景。 针对这些类型，Kubernetes 所执行的合法性检查操作以及对其所实施的限制各不相同。</p>
<table>
<thead>
<tr>
<th>内置类型</th>
<th>用法</th>
</tr>
</thead>
<tbody><tr>
<td><code>Opaque</code></td>
<td>用户定义的任意数据</td>
</tr>
<tr>
<td><code>kubernetes.io/service-account-token</code></td>
<td>服务账号令牌</td>
</tr>
<tr>
<td><code>kubernetes.io/dockercfg</code></td>
<td><code>~/.dockercfg</code> 文件的序列化形式</td>
</tr>
<tr>
<td><code>kubernetes.io/dockerconfigjson</code></td>
<td><code>~/.docker/config.json</code> 文件的序列化形式</td>
</tr>
<tr>
<td><code>kubernetes.io/basic-auth</code></td>
<td>用于基本身份认证的凭据</td>
</tr>
<tr>
<td><code>kubernetes.io/ssh-auth</code></td>
<td>用于 SSH 身份认证的凭据</td>
</tr>
<tr>
<td><code>kubernetes.io/tls</code></td>
<td>用于 TLS 客户端或者服务器端的数据</td>
</tr>
<tr>
<td><code>bootstrap.kubernetes.io/token</code></td>
<td>启动引导令牌数据</td>
</tr>
</tbody></table>
<p>通过为 Secret 对象的 <code>type</code> 字段设置一个非空的字符串值，你也可以定义并使用自己 Secret 类型。如果 <code>type</code> 值为空字符串，则被视为 <code>Opaque</code> 类型。 Kubernetes 并不对类型的名称作任何限制。不过，如果你要使用内置类型之一， 则你必须满足为该类型所定义的所有要求。</p>
<h3 id="2-1-1-Opaque-Secret"><a href="#2-1-1-Opaque-Secret" class="headerlink" title="2.1.1 Opaque Secret"></a>2.1.1 Opaque Secret</h3><p>当 Secret 配置文件中未作显式设定时，默认的 Secret 类型是 <code>Opaque</code>。 当你使用 <code>kubectl</code> 来创建一个 Secret 时，你会使用 <code>generic</code> 子命令来标明 要创建的是一个 <code>Opaque</code> 类型 Secret。 例如，下面的命令会创建一个空的 <code>Opaque</code> 类型 Secret 对象：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">kubectl create secret generic empty-secret</span><br><span class="line">kubectl get secret empty-secret</span><br></pre></td></tr></table></figure>

<p>输出类似于</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">NAME           TYPE     DATA   AGE</span><br><span class="line">empty-secret   Opaque   0      2m6s</span><br></pre></td></tr></table></figure>

<p><code>DATA</code> 列显示 Secret 中保存的数据条目个数。 在这个例子种，<code>0</code> 意味着我们刚刚创建了一个空的 Secret。</p>
<h3 id="2-1-2-Docker-配置-Secret"><a href="#2-1-2-Docker-配置-Secret" class="headerlink" title="2.1.2 Docker 配置 Secret"></a>2.1.2 Docker 配置 Secret</h3><p>你可以使用下面两种 <code>type</code> 值之一来创建 Secret，用以存放访问 Docker 仓库 来下载镜像的凭据。</p>
<ul>
<li><code>kubernetes.io/dockercfg</code></li>
<li><code>kubernetes.io/dockerconfigjson</code></li>
</ul>
<p><code>kubernetes.io/dockercfg</code> 是一种保留类型，用来存放 <code>~/.dockercfg</code> 文件的 序列化形式。该文件是配置 Docker 命令行的一种老旧形式。 使用此 Secret 类型时，你需要确保 Secret 的 <code>data</code> 字段中包含名为 <code>.dockercfg</code> 的主键，其对应键值是用 base64 编码的某 <code>~/.dockercfg</code> 文件的内容。</p>
<p>类型 <code>kubernetes.io/dockerconfigjson</code> 被设计用来保存 JSON 数据的序列化形式， 该 JSON 也遵从 <code>~/.docker/config.json</code> 文件的格式规则，而后者是 <code>~/.dockercfg</code> 的新版本格式。 使用此 Secret 类型时，Secret 对象的 <code>data</code> 字段必须包含 <code>.dockerconfigjson</code> 键，其键值为 base64 编码的字符串包含 <code>~/.docker/config.json</code> 文件的内容。</p>
<p>下面是一个 <code>kubernetes.io/dockercfg</code> 类型 Secret 的示例：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-dockercfg</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">kubernetes.io/dockercfg</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="string">.dockercfg:</span> <span class="string">|</span></span><br><span class="line">        <span class="string">&quot;&lt;base64 encoded ~/.dockercfg file&gt;&quot;</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>说明：</strong></p>
<p>如果你不希望执行 base64 编码转换，可以使用 <code>stringData</code> 字段代替。</p>
</blockquote>
<p>当你使用清单文件来创建这两类 Secret 时，API 服务器会检查 <code>data</code> 字段中是否 存在所期望的主键，并且验证其中所提供的键值是否是合法的 JSON 数据。 不过，API 服务器不会检查 JSON 数据本身是否是一个合法的 Docker 配置文件内容。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">kubectl create secret docker-registry secret-tiger-docker \</span><br><span class="line">  --docker-username=tiger \</span><br><span class="line">  --docker-password=pass113 \</span><br><span class="line">  --docker-email=tiger@acme.com</span><br></pre></td></tr></table></figure>

<p>上面的命令创建一个类型为 <code>kubernetes.io/dockerconfigjson</code> 的 Secret。 如果你对 <code>data</code> 字段中的 <code>.dockerconfigjson</code> 内容进行转储，你会得到下面的 JSON 内容，而这一内容是一个合法的 Docker 配置文件。</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">&quot;auths&quot;</span>: &#123;</span><br><span class="line">    <span class="attr">&quot;https://index.docker.io/v1/&quot;</span>: &#123;</span><br><span class="line">      <span class="attr">&quot;username&quot;</span>: <span class="string">&quot;tiger&quot;</span>,</span><br><span class="line">      <span class="attr">&quot;password&quot;</span>: <span class="string">&quot;pass113&quot;</span>,</span><br><span class="line">      <span class="attr">&quot;email&quot;</span>: <span class="string">&quot;tiger@acme.com&quot;</span>,</span><br><span class="line">      <span class="attr">&quot;auth&quot;</span>: <span class="string">&quot;dGlnZXI6cGFzczExMw==&quot;</span></span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>



<h3 id="2-1-3-基本身份认证-Secret"><a href="#2-1-3-基本身份认证-Secret" class="headerlink" title="2.1.3 基本身份认证 Secret"></a>2.1.3 基本身份认证 Secret</h3><p><code>kubernetes.io/basic-auth</code> 类型用来存放用于基本身份认证所需的凭据信息。 使用这种 Secret 类型时，Secret 的 <code>data</code> 字段必须包含以下两个键：</p>
<ul>
<li><code>username</code>: 用于身份认证的用户名；</li>
<li><code>password</code>: 用于身份认证的密码或令牌。</li>
</ul>
<p>以上两个键的键值都是 base64 编码的字符串。 当然你也可以在创建 Secret 时使用 <code>stringData</code> 字段来提供明文形式的内容。 下面的 YAML 是基本身份认证 Secret 的一个示例清单：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-basic-auth</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">kubernetes.io/basic-auth</span></span><br><span class="line"><span class="attr">stringData:</span></span><br><span class="line">  <span class="attr">username:</span> <span class="string">admin</span></span><br><span class="line">  <span class="attr">password:</span> <span class="string">t0p-Secret</span></span><br></pre></td></tr></table></figure>





<h2 id="2-2-创建Secret"><a href="#2-2-创建Secret" class="headerlink" title="2.2 创建Secret"></a>2.2 创建Secret</h2><h3 id="2-2-1-使用-kubectl-管理-Secret"><a href="#2-2-1-使用-kubectl-管理-Secret" class="headerlink" title="2.2.1 使用 kubectl 管理 Secret"></a>2.2.1 使用 kubectl 管理 Secret</h3><h4 id="2-2-1-1-创建-Secret"><a href="#2-2-1-1-创建-Secret" class="headerlink" title="2.2.1.1 创建 Secret"></a>2.2.1.1 创建 Secret</h4><p>一个 <code>Secret</code> 可以包含 Pod 访问数据库所需的用户凭证。 例如，由用户名和密码组成的数据库连接字符串。 你可以在本地计算机上，将用户名存储在文件 <code>./username.txt</code> 中，将密码存储在文件 <code>./password.txt</code> 中。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">echo -n &#x27;admin&#x27; &gt; ./username.txt</span><br><span class="line">echo -n &#x27;1f2d1e2e67df&#x27; &gt; ./password.txt</span><br></pre></td></tr></table></figure>

<p>上面两个命令中的 <code>-n</code> 标志确保生成的文件在文本末尾不包含额外的换行符。 这一点很重要，因为当 <code>kubectl</code> 读取文件并将内容编码为 base64 字符串时，多余的换行符也会被编码。</p>
<p><code>kubectl create secret</code> 命令将这些文件打包成一个 Secret 并在 API 服务器上创建对象。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubectl create secret generic db-user-pass \</span><br><span class="line">  --from-file=./username.txt \</span><br><span class="line">  --from-file=./password.txt</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">secret&#x2F;db-user-pass created</span><br></pre></td></tr></table></figure>

<p>默认密钥名称是文件名。 你可以选择使用 <code>--from-file=[key=]source</code> 来设置密钥名称。例如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubectl create secret generic db-user-pass \</span><br><span class="line">  --from-file=username=./username.txt \</span><br><span class="line">  --from-file=password=./password.txt</span><br></pre></td></tr></table></figure>

<p>你无需转义文件（<code>--from-file</code>）中的密码的特殊字符。</p>
<p>你还可以使用 <code>--from-literal=&lt;key&gt;=&lt;value&gt;</code> 标签提供 Secret 数据。 可以多次使用此标签，提供多个键值对。 请注意，特殊字符（例如：<code>$</code>，<code>\</code>，<code>*</code>，<code>=</code> 和 <code>!</code>）由你的 <a target="_blank" rel="noopener" href="https://en.wikipedia.org/wiki/Shell_(computing)">shell</a> 解释执行，而且需要转义。 在大多数 shell 中，转义密码最简便的方法是用单引号括起来。 比如，如果你的密码是 <code>S!B\*d$zDsb=</code>， 可以像下面一样执行命令：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubectl create secret generic dev-db-secret \</span><br><span class="line">  --from-literal=username=devuser \</span><br><span class="line">  --from-literal=password=&#x27;S!B\*d$zDsb=&#x27;</span><br></pre></td></tr></table></figure>

<h4 id="2-2-1-2-验证-Secret"><a href="#2-2-1-2-验证-Secret" class="headerlink" title="2.2.1.2 验证 Secret"></a>2.2.1.2 验证 Secret</h4><p>你可以检查 secret 是否已创建：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get secrets</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">NAME                  TYPE                                  DATA      AGE</span><br><span class="line">db-user-pass          Opaque                                2         51s</span><br></pre></td></tr></table></figure>

<p>你可以查看 <code>Secret</code> 的描述：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl describe secrets/db-user-pass</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">Name:            db-user-pass</span><br><span class="line">Namespace:       default</span><br><span class="line">Labels:          &lt;none&gt;</span><br><span class="line">Annotations:     &lt;none&gt;</span><br><span class="line"></span><br><span class="line">Type:            Opaque</span><br><span class="line"></span><br><span class="line">Data</span><br><span class="line">&#x3D;&#x3D;&#x3D;&#x3D;</span><br><span class="line">password:    12 bytes</span><br><span class="line">username:    5 bytes</span><br></pre></td></tr></table></figure>

<p><code>kubectl get</code> 和 <code>kubectl describe</code> 命令默认不显示 <code>Secret</code> 的内容。 这是为了防止 <code>Secret</code> 被意外暴露给旁观者或存储在终端日志中。</p>
<h4 id="2-2-1-3-解码-Secret"><a href="#2-2-1-3-解码-Secret" class="headerlink" title="2.2.1.3 解码 Secret"></a>2.2.1.3 解码 Secret</h4><p>要查看我们刚刚创建的 Secret 的内容，可以运行以下命令：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get secret db-user-pass -o jsonpath=&#x27;&#123;.data&#125;&#x27;</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">&#123;<span class="attr">&quot;password.txt&quot;</span>:<span class="string">&quot;MWYyZDFlMmU2N2Rm&quot;</span>,<span class="attr">&quot;username.txt&quot;</span>:<span class="string">&quot;YWRtaW4=&quot;</span>&#125;</span><br></pre></td></tr></table></figure>

<p>现在你可以解码 <code>password.txt</code> 的数据：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">echo &#x27;MWYyZDFlMmU2N2Rm&#x27; | base64 --decode</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">1f2d1e2e67df</span><br></pre></td></tr></table></figure>



<h4 id="2-2-1-4-清理"><a href="#2-2-1-4-清理" class="headerlink" title="2.2.1.4 清理"></a>2.2.1.4 清理</h4><p>删除刚刚创建的 Secret：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete secret db-user-pass</span><br></pre></td></tr></table></figure>







<h3 id="2-2-2-使用配置文件管理-Secret"><a href="#2-2-2-使用配置文件管理-Secret" class="headerlink" title="2.2.2 使用配置文件管理 Secret"></a>2.2.2 使用配置文件管理 Secret</h3><h4 id="2-2-2-1-创建配置文件"><a href="#2-2-2-1-创建配置文件" class="headerlink" title="2.2.2.1 创建配置文件"></a>2.2.2.1 创建配置文件</h4><p>你可以先用 JSON 或 YAML 格式在文件中创建 Secret，然后创建该对象。 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secret-v1-core">Secret</a> 资源包含2个键值对： <code>data</code> 和 <code>stringData</code>。 <code>data</code> 字段用来存储 base64 编码的任意数据。 提供 <code>stringData</code> 字段是为了方便，它允许 Secret 使用未编码的字符串。 <code>data</code> 和 <code>stringData</code> 的键必须由字母、数字、<code>-</code>，<code>_</code> 或 <code>.</code> 组成。</p>
<p>例如，要使用 Secret 的 <code>data</code> 字段存储两个字符串，请将字符串转换为 base64 ，如下所示：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">echo -n &#x27;admin&#x27; | base64</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">YWRtaW4&#x3D;</span><br><span class="line">echo -n &#39;1f2d1e2e67df&#39; | base64</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">MWYyZDFlMmU2N2Rm</span><br></pre></td></tr></table></figure>

<p>编写一个 Secret 配置文件，如下所示：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">Opaque</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">username:</span> <span class="string">YWRtaW4=</span></span><br><span class="line">  <span class="attr">password:</span> <span class="string">MWYyZDFlMmU2N2Rm</span></span><br></pre></td></tr></table></figure>

<p>注意，Secret 对象的名称必须是有效的 <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/names#dns-subdomain-names">DNS 子域名</a>.</p>
<blockquote>
<p><strong>说明：</strong></p>
<p>Secret 数据的 JSON 和 YAML 序列化结果是以 base64 编码的。 换行符在这些字符串中无效，必须省略。 在 Darwin/macOS 上使用 <code>base64</code> 工具时，用户不应该使用 <code>-b</code> 选项分割长行。 相反地，Linux 用户 <em>应该</em> 在 <code>base64</code> 地命令中添加 <code>-w 0</code> 选项， 或者在 <code>-w</code> 选项不可用的情况下，输入 <code>base64 | tr -d &#39;\n&#39;</code>。</p>
</blockquote>
<p>对于某些场景，你可能希望使用 <code>stringData</code> 字段。 这字段可以将一个非 base64 编码的字符串直接放入 Secret 中， 当创建或更新该 Secret 时，此字段将被编码。</p>
<p>上述用例的实际场景可能是这样：当你部署应用时，使用 Secret 存储配置文件， 你希望在部署过程中，填入部分内容到该配置文件。</p>
<p>例如，如果你的应用程序使用以下配置文件:</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiUrl:</span> <span class="string">&quot;https://my.api.com/api/v1&quot;</span></span><br><span class="line"><span class="attr">username:</span> <span class="string">&quot;&lt;user&gt;&quot;</span></span><br><span class="line"><span class="attr">password:</span> <span class="string">&quot;&lt;password&gt;&quot;</span></span><br></pre></td></tr></table></figure>

<p>你可以使用以下定义将其存储在 Secret 中:</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">Opaque</span></span><br><span class="line"><span class="attr">stringData:</span></span><br><span class="line">  <span class="attr">config.yaml:</span> <span class="string">|</span></span><br><span class="line">    <span class="attr">apiUrl:</span> <span class="string">&quot;https://my.api.com/api/v1&quot;</span></span><br><span class="line">    <span class="attr">username:</span> <span class="string">&lt;user&gt;</span></span><br><span class="line">    <span class="attr">password:</span> <span class="string">&lt;password&gt;</span>    </span><br></pre></td></tr></table></figure>

<h4 id="2-2-2-2-创建-Secret-对象"><a href="#2-2-2-2-创建-Secret-对象" class="headerlink" title="2.2.2.2 创建 Secret 对象"></a>2.2.2.2 创建 Secret 对象</h4><p>现在使用 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply"><code>kubectl apply</code></a> 创建 Secret：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f ./secret.yaml</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">secret&#x2F;mysecret created</span><br></pre></td></tr></table></figure>



<h4 id="2-2-2-3-检查-Secret"><a href="#2-2-2-3-检查-Secret" class="headerlink" title="2.2.2.3 检查 Secret"></a>2.2.2.3 检查 Secret</h4><p><code>stringData</code> 字段是只写的。获取 Secret 时，此字段永远不会输出。 例如，如果你运行以下命令：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get secret mysecret -o yaml</span><br></pre></td></tr></table></figure>

<p>输出类似于：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">creationTimestamp:</span> <span class="number">2018-11-15T20:40:59Z</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">resourceVersion:</span> <span class="string">&quot;7225&quot;</span></span><br><span class="line">  <span class="attr">uid:</span> <span class="string">c280ad2e-e916-11e8-98f2-025000000001</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">Opaque</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">config.yaml:</span> <span class="string">YXBpVXJsOiAiaHR0cHM6Ly9teS5hcGkuY29tL2FwaS92MSIKdXNlcm5hbWU6IHt7dXNlcm5hbWV9fQpwYXNzd29yZDoge3twYXNzd29yZH19</span></span><br></pre></td></tr></table></figure>

<p>命令 <code>kubectl get</code> 和 <code>kubectl describe</code> 默认不显示 <code>Secret</code> 的内容。 这是为了防止 <code>Secret</code> 意外地暴露给旁观者或者保存在终端日志中。 检查编码数据的实际内容，请参考<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/tasks/configmap-secret/managing-secret-using-kubectl/#decoding-secret">解码 secret</a>.</p>
<p>如果在 <code>data</code> 和 <code>stringData</code> 中都指定了一个字段，比如 <code>username</code>，字段值来自 <code>stringData</code>。 例如，下面的 Secret 定义:</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">Opaque</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">username:</span> <span class="string">YWRtaW4=</span></span><br><span class="line"><span class="attr">stringData:</span></span><br><span class="line">  <span class="attr">username:</span> <span class="string">administrator</span></span><br></pre></td></tr></table></figure>

<p>结果有以下 Secret：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">creationTimestamp:</span> <span class="number">2018-11-15T20:46:46Z</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line">  <span class="attr">resourceVersion:</span> <span class="string">&quot;7579&quot;</span></span><br><span class="line">  <span class="attr">uid:</span> <span class="string">91460ecb-e917-11e8-98f2-025000000001</span></span><br><span class="line"><span class="attr">type:</span> <span class="string">Opaque</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">username:</span> <span class="string">YWRtaW5pc3RyYXRvcg==</span></span><br></pre></td></tr></table></figure>

<p>其中 <code>YWRtaW5pc3RyYXRvcg==</code> 解码成 <code>administrator</code>。</p>
<h4 id="2-2-2-4-清理"><a href="#2-2-2-4-清理" class="headerlink" title="2.2.2.4 清理"></a>2.2.2.4 清理</h4><p>删除你刚才创建的 Secret：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete secret mysecret</span><br></pre></td></tr></table></figure>









































<h2 id="2-3-使用-Secret"><a href="#2-3-使用-Secret" class="headerlink" title="2.3 使用 Secret"></a>2.3 使用 Secret</h2><p>Secret 可以作为数据卷被挂载，或作为<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/containers/container-environment/">环境变量</a> 暴露出来以供 Pod 中的容器使用。它们也可以被系统的其他部分使用，而不直接暴露在 Pod 内。 例如，它们可以保存凭据，系统的其他部分将用它来代表你与外部系统进行交互。</p>
<h3 id="2-3-1-在-Pod-中使用-Secret-文件"><a href="#2-3-1-在-Pod-中使用-Secret-文件" class="headerlink" title="2.3.1 在 Pod 中使用 Secret 文件"></a>2.3.1 在 Pod 中使用 Secret 文件</h3><p>在 Pod 中使用存放在卷中的 Secret：</p>
<ol>
<li>创建一个 Secret 或者使用已有的 Secret。多个 Pod 可以引用同一个 Secret。</li>
<li>修改你的 Pod 定义，在 <code>spec.volumes[]</code> 下增加一个卷。可以给这个卷随意命名， 它的 <code>spec.volumes[].secret.secretName</code> 必须是 Secret 对象的名字。</li>
<li>将 <code>spec.containers[].volumeMounts[]</code> 加到需要用到该 Secret 的容器中。 指定 <code>spec.containers[].volumeMounts[].readOnly = true</code> 和 <code>spec.containers[].volumeMounts[].mountPath</code> 为你想要该 Secret 出现的尚未使用的目录。</li>
<li>修改你的镜像并且／或者命令行，让程序从该目录下寻找文件。 Secret 的 <code>data</code> 映射中的每一个键都对应 <code>mountPath</code> 下的一个文件名。</li>
</ol>
<p>这是一个在 Pod 中使用存放在挂载卷中 Secret 的例子：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mypod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypod</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">redis</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">foo</span></span><br><span class="line">      <span class="attr">mountPath:</span> <span class="string">&quot;/etc/foo&quot;</span></span><br><span class="line">      <span class="attr">readOnly:</span> <span class="literal">true</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">foo</span></span><br><span class="line">    <span class="attr">secret:</span></span><br><span class="line">      <span class="attr">secretName:</span> <span class="string">mysecret</span></span><br></pre></td></tr></table></figure>

<p>您想要用的每个 Secret 都需要在 <code>spec.volumes</code> 中引用。</p>
<p>如果 Pod 中有多个容器，每个容器都需要自己的 <code>volumeMounts</code> 配置块， 但是每个 Secret 只需要一个 <code>spec.volumes</code>。</p>
<p>您可以打包多个文件到一个 Secret 中，或者使用的多个 Secret，怎样方便就怎样来。</p>
<h3 id="2-3-2-将-Secret-键名映射到特定路径"><a href="#2-3-2-将-Secret-键名映射到特定路径" class="headerlink" title="2.3.2 将 Secret 键名映射到特定路径"></a>2.3.2 将 Secret 键名映射到特定路径</h3><p>我们还可以控制 Secret 键名在存储卷中映射的的路径。 你可以使用 <code>spec.volumes[].secret.items</code> 字段修改每个键对应的目标路径：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">mypod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypod</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">redis</span></span><br><span class="line">    <span class="attr">volumeMounts:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">foo</span></span><br><span class="line">      <span class="attr">mountPath:</span> <span class="string">&quot;/etc/foo&quot;</span></span><br><span class="line">      <span class="attr">readOnly:</span> <span class="literal">true</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">foo</span></span><br><span class="line">    <span class="attr">secret:</span></span><br><span class="line">      <span class="attr">secretName:</span> <span class="string">mysecret</span></span><br><span class="line">      <span class="attr">items:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">key:</span> <span class="string">username</span></span><br><span class="line">        <span class="attr">path:</span> <span class="string">my-group/my-username</span></span><br></pre></td></tr></table></figure>

<p>将会发生什么呢：</p>
<ul>
<li><code>username</code> Secret 存储在 <code>/etc/foo/my-group/my-username</code> 文件中而不是 <code>/etc/foo/username</code> 中。</li>
<li><code>password</code> Secret 没有被映射</li>
</ul>
<p>如果使用了 <code>spec.volumes[].secret.items</code>，只有在 <code>items</code> 中指定的键会被映射。 要使用 Secret 中所有键，就必须将它们都列在 <code>items</code> 字段中。 所有列出的键名必须存在于相应的 Secret 中。否则，不会创建卷。</p>
<h3 id="2-3-3-挂载的-Secret-会被自动更新"><a href="#2-3-3-挂载的-Secret-会被自动更新" class="headerlink" title="2.3.3 挂载的 Secret 会被自动更新"></a>2.3.3 挂载的 Secret 会被自动更新</h3><p>当已经存储于卷中被使用的 Secret 被更新时，被映射的键也将终将被更新。 组件 kubelet 在周期性同步时检查被挂载的 Secret 是不是最新的。 但是，它会使用其本地缓存的数值作为 Secret 的当前值。</p>
<p>缓存的类型可以使用 <a target="_blank" rel="noopener" href="https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go">KubeletConfiguration 结构</a> 中的 <code>ConfigMapAndSecretChangeDetectionStrategy</code> 字段来配置。 它可以通过 watch 操作来传播（默认），基于 TTL 来刷新，也可以 将所有请求直接重定向到 API 服务器。 因此，从 Secret 被更新到将新 Secret 被投射到 Pod 的那一刻的总延迟可能与 kubelet 同步周期 + 缓存传播延迟一样长，其中缓存传播延迟取决于所选的缓存类型。 对应于不同的缓存类型，该延迟或者等于 watch 传播延迟，或者等于缓存的 TTL， 或者为 0。</p>
<blockquote>
<p><strong>说明：</strong> 使用 Secret 作为<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/storage/volumes#using-subpath">子路径</a>卷挂载的容器 不会收到 Secret 更新。</p>
</blockquote>
<h3 id="2-3-4-以环境变量的形式使用-Secrets"><a href="#2-3-4-以环境变量的形式使用-Secrets" class="headerlink" title="2.3.4 以环境变量的形式使用 Secrets"></a>2.3.4 以环境变量的形式使用 Secrets</h3><p>将 Secret 作为 Pod 中的<a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/concepts/containers/container-environment/">环境变量</a>使用：</p>
<ol>
<li>创建一个 Secret 或者使用一个已存在的 Secret。多个 Pod 可以引用同一个 Secret。</li>
<li>修改 Pod 定义，为每个要使用 Secret 的容器添加对应 Secret 键的环境变量。 使用 Secret 键的环境变量应在 <code>env[x].valueFrom.secretKeyRef</code> 中指定 要包含的 Secret 名称和键名。</li>
<li>更改镜像并／或者命令行，以便程序在指定的环境变量中查找值。</li>
</ol>
<p>这是一个使用来自环境变量中的 Secret 值的 Pod 示例：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">secret-env-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mycontainer</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">redis</span></span><br><span class="line">    <span class="attr">env:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SECRET_USERNAME</span></span><br><span class="line">        <span class="attr">valueFrom:</span></span><br><span class="line">          <span class="attr">secretKeyRef:</span></span><br><span class="line">            <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line">            <span class="attr">key:</span> <span class="string">username</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">SECRET_PASSWORD</span></span><br><span class="line">        <span class="attr">valueFrom:</span></span><br><span class="line">          <span class="attr">secretKeyRef:</span></span><br><span class="line">            <span class="attr">name:</span> <span class="string">mysecret</span></span><br><span class="line">            <span class="attr">key:</span> <span class="string">password</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Never</span></span><br></pre></td></tr></table></figure>

<h4 id="注：-Secret-更新之后对应的环境变量不会被更新"><a href="#注：-Secret-更新之后对应的环境变量不会被更新" class="headerlink" title="注： Secret 更新之后对应的环境变量不会被更新"></a>注： Secret 更新之后对应的环境变量不会被更新</h4><p>如果某个容器已经在通过环境变量使用某 Secret，对该 Secret 的更新不会被 容器马上看见，除非容器被重启。有一些第三方的解决方案能够在 Secret 发生 变化时触发容器重启。</p>
<h3 id="2-3-5-不可更改的-Secret"><a href="#2-3-5-不可更改的-Secret" class="headerlink" title="2.3.5 不可更改的 Secret"></a>2.3.5 不可更改的 Secret</h3><p><strong>FEATURE STATE:</strong> <code>Kubernetes v1.19 [beta]</code></p>
<p>Kubernetes 的 alpha 特性 <em>不可变的 Secret 和 ConfigMap</em> 提供了一种可选配置， 可以设置各个 Secret 和 ConfigMap 为不可变的。 对于大量使用 Secret 的集群（至少有成千上万各不相同的 Secret 供 Pod 挂载）， 禁止变更它们的数据有下列好处：</p>
<ul>
<li>防止意外（或非预期的）更新导致应用程序中断</li>
<li>通过将 Secret 标记为不可变来关闭 kube-apiserver 对其的监视，从而<strong>显著降低 kube-apiserver 的负载，提升集群性能</strong>。</li>
</ul>
<p>这个特性通过 <code>ImmutableEmphemeralVolumes</code> <a target="_blank" rel="noopener" href="https://kubernetes.io/zh/docs/reference/command-line-tools-reference/feature-gates/">特性门控</a> 来控制，从 v1.19 开始默认启用。 你可以通过将 Secret 的 <code>immutable</code> 字段设置为 <code>true</code> 创建不可更改的 Secret。 例如：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="string">...</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="string">...</span></span><br><span class="line"><span class="attr">immutable:</span> <span class="literal">true</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>说明：</strong></p>
<p>一旦一个 Secret 或 ConfigMap 被标记为不可更改，撤销此操作或者更改 <code>data</code> 字段的内容都是 <em>不</em> 可能的。 只能删除并重新创建这个 Secret。现有的 Pod 将维持对已删除 Secret 的挂载点 - 建议重新创建这些 Pod。</p>
</blockquote>
<p>=======</p>
<blockquote>
<blockquote>
<blockquote>
<blockquote>
<blockquote>
<blockquote>
<blockquote>
<p>cfe56dad9493312e3a2e2b7721e829af8f7a9f1d</p>
</blockquote>
</blockquote>
</blockquote>
</blockquote>
</blockquote>
</blockquote>
</blockquote>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">Configmap</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color2">Secret</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80ConfigMap%20%E5%92%8C%20Secret/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
  
    <nav id="page-nav">
      <a class="extend prev" rel="prev" href="/">&amp;laquo; Prev</a><a class="page-number" href="/">1</a><span class="page-number current">2</span><a class="page-number" href="/page/3/">3</a><a class="page-number" href="/page/4/">4</a><span class="space">&hellip;</span><a class="page-number" href="/page/6/">6</a><a class="extend next" rel="next" href="/page/3/">Next &amp;raquo;</a>
    </nav>
  


          </div>
        </div>
      </div>
      <footer id="footer">
  <div class="outer">
    <div id="footer-info">
    	<div class="footer-left">
    		&copy; 2021 宸轩少翁
    	</div>
      	<div class="footer-right">
      		<a href="http://hexo.io/" target="_blank">Hexo</a>  Theme <a href="https://github.com/litten/hexo-theme-yilia" target="_blank">Yilia</a> by Litten
      	</div>
    </div>
	        <!-- 不蒜子统计 -->            
	   <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js">
	   </script>	      
		本站总访问量<span id="busuanzi_value_site_pv"></span>次   
  </div>
</footer>
    </div>
    <script>
	var yiliaConfig = {
		mathjax: false,
		isHome: true,
		isPost: false,
		isArchive: false,
		isTag: false,
		isCategory: false,
		open_in_new: false,
		toc_hide_index: true,
		root: "/",
		innerArchive: true,
		showTags: false
	}
</script>

<script>!function(t){function n(e){if(r[e])return r[e].exports;var i=r[e]={exports:{},id:e,loaded:!1};return t[e].call(i.exports,i,i.exports,n),i.loaded=!0,i.exports}var r={};n.m=t,n.c=r,n.p="./",n(0)}([function(t,n,r){r(195),t.exports=r(191)},function(t,n,r){var e=r(3),i=r(52),o=r(27),u=r(28),c=r(53),f="prototype",a=function(t,n,r){var s,l,h,v,p=t&a.F,d=t&a.G,y=t&a.S,g=t&a.P,b=t&a.B,m=d?e:y?e[n]||(e[n]={}):(e[n]||{})[f],x=d?i:i[n]||(i[n]={}),w=x[f]||(x[f]={});d&&(r=n);for(s in r)l=!p&&m&&void 0!==m[s],h=(l?m:r)[s],v=b&&l?c(h,e):g&&"function"==typeof h?c(Function.call,h):h,m&&u(m,s,h,t&a.U),x[s]!=h&&o(x,s,v),g&&w[s]!=h&&(w[s]=h)};e.core=i,a.F=1,a.G=2,a.S=4,a.P=8,a.B=16,a.W=32,a.U=64,a.R=128,t.exports=a},function(t,n,r){var e=r(6);t.exports=function(t){if(!e(t))throw TypeError(t+" is not an object!");return t}},function(t,n){var r=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=r)},function(t,n){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,n){var r=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=r)},function(t,n){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,n,r){var e=r(126)("wks"),i=r(76),o=r(3).Symbol,u="function"==typeof o;(t.exports=function(t){return e[t]||(e[t]=u&&o[t]||(u?o:i)("Symbol."+t))}).store=e},function(t,n){var r={}.hasOwnProperty;t.exports=function(t,n){return r.call(t,n)}},function(t,n,r){var e=r(94),i=r(33);t.exports=function(t){return e(i(t))}},function(t,n,r){t.exports=!r(4)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(2),i=r(167),o=r(50),u=Object.defineProperty;n.f=r(10)?Object.defineProperty:function(t,n,r){if(e(t),n=o(n,!0),e(r),i)try{return u(t,n,r)}catch(t){}if("get"in r||"set"in r)throw TypeError("Accessors not supported!");return"value"in r&&(t[n]=r.value),t}},function(t,n,r){t.exports=!r(18)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(14),i=r(22);t.exports=r(12)?function(t,n,r){return e.f(t,n,i(1,r))}:function(t,n,r){return t[n]=r,t}},function(t,n,r){var e=r(20),i=r(58),o=r(42),u=Object.defineProperty;n.f=r(12)?Object.defineProperty:function(t,n,r){if(e(t),n=o(n,!0),e(r),i)try{return u(t,n,r)}catch(t){}if("get"in r||"set"in r)throw TypeError("Accessors not supported!");return"value"in r&&(t[n]=r.value),t}},function(t,n,r){var e=r(40)("wks"),i=r(23),o=r(5).Symbol,u="function"==typeof o;(t.exports=function(t){return e[t]||(e[t]=u&&o[t]||(u?o:i)("Symbol."+t))}).store=e},function(t,n,r){var e=r(67),i=Math.min;t.exports=function(t){return t>0?i(e(t),9007199254740991):0}},function(t,n,r){var e=r(46);t.exports=function(t){return Object(e(t))}},function(t,n){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,n,r){var e=r(63),i=r(34);t.exports=Object.keys||function(t){return e(t,i)}},function(t,n,r){var e=r(21);t.exports=function(t){if(!e(t))throw TypeError(t+" is not an object!");return t}},function(t,n){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,n){t.exports=function(t,n){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:n}}},function(t,n){var r=0,e=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++r+e).toString(36))}},function(t,n){var r={}.hasOwnProperty;t.exports=function(t,n){return r.call(t,n)}},function(t,n){var r=t.exports={version:"2.4.0"};"number"==typeof __e&&(__e=r)},function(t,n){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,n,r){var e=r(11),i=r(66);t.exports=r(10)?function(t,n,r){return e.f(t,n,i(1,r))}:function(t,n,r){return t[n]=r,t}},function(t,n,r){var e=r(3),i=r(27),o=r(24),u=r(76)("src"),c="toString",f=Function[c],a=(""+f).split(c);r(52).inspectSource=function(t){return f.call(t)},(t.exports=function(t,n,r,c){var f="function"==typeof r;f&&(o(r,"name")||i(r,"name",n)),t[n]!==r&&(f&&(o(r,u)||i(r,u,t[n]?""+t[n]:a.join(String(n)))),t===e?t[n]=r:c?t[n]?t[n]=r:i(t,n,r):(delete t[n],i(t,n,r)))})(Function.prototype,c,function(){return"function"==typeof this&&this[u]||f.call(this)})},function(t,n,r){var e=r(1),i=r(4),o=r(46),u=function(t,n,r,e){var i=String(o(t)),u="<"+n;return""!==r&&(u+=" "+r+'="'+String(e).replace(/"/g,"&quot;")+'"'),u+">"+i+"</"+n+">"};t.exports=function(t,n){var r={};r[t]=n(u),e(e.P+e.F*i(function(){var n=""[t]('"');return n!==n.toLowerCase()||n.split('"').length>3}),"String",r)}},function(t,n,r){var e=r(115),i=r(46);t.exports=function(t){return e(i(t))}},function(t,n,r){var e=r(116),i=r(66),o=r(30),u=r(50),c=r(24),f=r(167),a=Object.getOwnPropertyDescriptor;n.f=r(10)?a:function(t,n){if(t=o(t),n=u(n,!0),f)try{return a(t,n)}catch(t){}if(c(t,n))return i(!e.f.call(t,n),t[n])}},function(t,n,r){var e=r(24),i=r(17),o=r(145)("IE_PROTO"),u=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=i(t),e(t,o)?t[o]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?u:null}},function(t,n){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,n){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,n){t.exports={}},function(t,n){t.exports=!0},function(t,n){n.f={}.propertyIsEnumerable},function(t,n,r){var e=r(14).f,i=r(8),o=r(15)("toStringTag");t.exports=function(t,n,r){t&&!i(t=r?t:t.prototype,o)&&e(t,o,{configurable:!0,value:n})}},function(t,n,r){var e=r(40)("keys"),i=r(23);t.exports=function(t){return e[t]||(e[t]=i(t))}},function(t,n,r){var e=r(5),i="__core-js_shared__",o=e[i]||(e[i]={});t.exports=function(t){return o[t]||(o[t]={})}},function(t,n){var r=Math.ceil,e=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?e:r)(t)}},function(t,n,r){var e=r(21);t.exports=function(t,n){if(!e(t))return t;var r,i;if(n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;if("function"==typeof(r=t.valueOf)&&!e(i=r.call(t)))return i;if(!n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;throw TypeError("Can't convert object to primitive value")}},function(t,n,r){var e=r(5),i=r(25),o=r(36),u=r(44),c=r(14).f;t.exports=function(t){var n=i.Symbol||(i.Symbol=o?{}:e.Symbol||{});"_"==t.charAt(0)||t in n||c(n,t,{value:u.f(t)})}},function(t,n,r){n.f=r(15)},function(t,n){var r={}.toString;t.exports=function(t){return r.call(t).slice(8,-1)}},function(t,n){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,n,r){var e=r(4);t.exports=function(t,n){return!!t&&e(function(){n?t.call(null,function(){},1):t.call(null)})}},function(t,n,r){var e=r(53),i=r(115),o=r(17),u=r(16),c=r(203);t.exports=function(t,n){var r=1==t,f=2==t,a=3==t,s=4==t,l=6==t,h=5==t||l,v=n||c;return function(n,c,p){for(var d,y,g=o(n),b=i(g),m=e(c,p,3),x=u(b.length),w=0,S=r?v(n,x):f?v(n,0):void 0;x>w;w++)if((h||w in b)&&(d=b[w],y=m(d,w,g),t))if(r)S[w]=y;else if(y)switch(t){case 3:return!0;case 5:return d;case 6:return w;case 2:S.push(d)}else if(s)return!1;return l?-1:a||s?s:S}}},function(t,n,r){var e=r(1),i=r(52),o=r(4);t.exports=function(t,n){var r=(i.Object||{})[t]||Object[t],u={};u[t]=n(r),e(e.S+e.F*o(function(){r(1)}),"Object",u)}},function(t,n,r){var e=r(6);t.exports=function(t,n){if(!e(t))return t;var r,i;if(n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;if("function"==typeof(r=t.valueOf)&&!e(i=r.call(t)))return i;if(!n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;throw TypeError("Can't convert object to primitive value")}},function(t,n,r){var e=r(5),i=r(25),o=r(91),u=r(13),c="prototype",f=function(t,n,r){var a,s,l,h=t&f.F,v=t&f.G,p=t&f.S,d=t&f.P,y=t&f.B,g=t&f.W,b=v?i:i[n]||(i[n]={}),m=b[c],x=v?e:p?e[n]:(e[n]||{})[c];v&&(r=n);for(a in r)(s=!h&&x&&void 0!==x[a])&&a in b||(l=s?x[a]:r[a],b[a]=v&&"function"!=typeof x[a]?r[a]:y&&s?o(l,e):g&&x[a]==l?function(t){var n=function(n,r,e){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(n);case 2:return new t(n,r)}return new t(n,r,e)}return t.apply(this,arguments)};return n[c]=t[c],n}(l):d&&"function"==typeof l?o(Function.call,l):l,d&&((b.virtual||(b.virtual={}))[a]=l,t&f.R&&m&&!m[a]&&u(m,a,l)))};f.F=1,f.G=2,f.S=4,f.P=8,f.B=16,f.W=32,f.U=64,f.R=128,t.exports=f},function(t,n){var r=t.exports={version:"2.4.0"};"number"==typeof __e&&(__e=r)},function(t,n,r){var e=r(26);t.exports=function(t,n,r){if(e(t),void 0===n)return t;switch(r){case 1:return function(r){return t.call(n,r)};case 2:return function(r,e){return t.call(n,r,e)};case 3:return function(r,e,i){return t.call(n,r,e,i)}}return function(){return t.apply(n,arguments)}}},function(t,n,r){var e=r(183),i=r(1),o=r(126)("metadata"),u=o.store||(o.store=new(r(186))),c=function(t,n,r){var i=u.get(t);if(!i){if(!r)return;u.set(t,i=new e)}var o=i.get(n);if(!o){if(!r)return;i.set(n,o=new e)}return o},f=function(t,n,r){var e=c(n,r,!1);return void 0!==e&&e.has(t)},a=function(t,n,r){var e=c(n,r,!1);return void 0===e?void 0:e.get(t)},s=function(t,n,r,e){c(r,e,!0).set(t,n)},l=function(t,n){var r=c(t,n,!1),e=[];return r&&r.forEach(function(t,n){e.push(n)}),e},h=function(t){return void 0===t||"symbol"==typeof t?t:String(t)},v=function(t){i(i.S,"Reflect",t)};t.exports={store:u,map:c,has:f,get:a,set:s,keys:l,key:h,exp:v}},function(t,n,r){"use strict";if(r(10)){var e=r(69),i=r(3),o=r(4),u=r(1),c=r(127),f=r(152),a=r(53),s=r(68),l=r(66),h=r(27),v=r(73),p=r(67),d=r(16),y=r(75),g=r(50),b=r(24),m=r(180),x=r(114),w=r(6),S=r(17),_=r(137),O=r(70),E=r(32),P=r(71).f,j=r(154),F=r(76),M=r(7),A=r(48),N=r(117),T=r(146),I=r(155),k=r(80),L=r(123),R=r(74),C=r(130),D=r(160),U=r(11),W=r(31),G=U.f,B=W.f,V=i.RangeError,z=i.TypeError,q=i.Uint8Array,K="ArrayBuffer",J="Shared"+K,Y="BYTES_PER_ELEMENT",H="prototype",$=Array[H],X=f.ArrayBuffer,Q=f.DataView,Z=A(0),tt=A(2),nt=A(3),rt=A(4),et=A(5),it=A(6),ot=N(!0),ut=N(!1),ct=I.values,ft=I.keys,at=I.entries,st=$.lastIndexOf,lt=$.reduce,ht=$.reduceRight,vt=$.join,pt=$.sort,dt=$.slice,yt=$.toString,gt=$.toLocaleString,bt=M("iterator"),mt=M("toStringTag"),xt=F("typed_constructor"),wt=F("def_constructor"),St=c.CONSTR,_t=c.TYPED,Ot=c.VIEW,Et="Wrong length!",Pt=A(1,function(t,n){return Tt(T(t,t[wt]),n)}),jt=o(function(){return 1===new q(new Uint16Array([1]).buffer)[0]}),Ft=!!q&&!!q[H].set&&o(function(){new q(1).set({})}),Mt=function(t,n){if(void 0===t)throw z(Et);var r=+t,e=d(t);if(n&&!m(r,e))throw V(Et);return e},At=function(t,n){var r=p(t);if(r<0||r%n)throw V("Wrong offset!");return r},Nt=function(t){if(w(t)&&_t in t)return t;throw z(t+" is not a typed array!")},Tt=function(t,n){if(!(w(t)&&xt in t))throw z("It is not a typed array constructor!");return new t(n)},It=function(t,n){return kt(T(t,t[wt]),n)},kt=function(t,n){for(var r=0,e=n.length,i=Tt(t,e);e>r;)i[r]=n[r++];return i},Lt=function(t,n,r){G(t,n,{get:function(){return this._d[r]}})},Rt=function(t){var n,r,e,i,o,u,c=S(t),f=arguments.length,s=f>1?arguments[1]:void 0,l=void 0!==s,h=j(c);if(void 0!=h&&!_(h)){for(u=h.call(c),e=[],n=0;!(o=u.next()).done;n++)e.push(o.value);c=e}for(l&&f>2&&(s=a(s,arguments[2],2)),n=0,r=d(c.length),i=Tt(this,r);r>n;n++)i[n]=l?s(c[n],n):c[n];return i},Ct=function(){for(var t=0,n=arguments.length,r=Tt(this,n);n>t;)r[t]=arguments[t++];return r},Dt=!!q&&o(function(){gt.call(new q(1))}),Ut=function(){return gt.apply(Dt?dt.call(Nt(this)):Nt(this),arguments)},Wt={copyWithin:function(t,n){return D.call(Nt(this),t,n,arguments.length>2?arguments[2]:void 0)},every:function(t){return rt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},fill:function(t){return C.apply(Nt(this),arguments)},filter:function(t){return It(this,tt(Nt(this),t,arguments.length>1?arguments[1]:void 0))},find:function(t){return et(Nt(this),t,arguments.length>1?arguments[1]:void 0)},findIndex:function(t){return it(Nt(this),t,arguments.length>1?arguments[1]:void 0)},forEach:function(t){Z(Nt(this),t,arguments.length>1?arguments[1]:void 0)},indexOf:function(t){return ut(Nt(this),t,arguments.length>1?arguments[1]:void 0)},includes:function(t){return ot(Nt(this),t,arguments.length>1?arguments[1]:void 0)},join:function(t){return vt.apply(Nt(this),arguments)},lastIndexOf:function(t){return st.apply(Nt(this),arguments)},map:function(t){return Pt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},reduce:function(t){return lt.apply(Nt(this),arguments)},reduceRight:function(t){return ht.apply(Nt(this),arguments)},reverse:function(){for(var t,n=this,r=Nt(n).length,e=Math.floor(r/2),i=0;i<e;)t=n[i],n[i++]=n[--r],n[r]=t;return n},some:function(t){return nt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},sort:function(t){return pt.call(Nt(this),t)},subarray:function(t,n){var r=Nt(this),e=r.length,i=y(t,e);return new(T(r,r[wt]))(r.buffer,r.byteOffset+i*r.BYTES_PER_ELEMENT,d((void 0===n?e:y(n,e))-i))}},Gt=function(t,n){return It(this,dt.call(Nt(this),t,n))},Bt=function(t){Nt(this);var n=At(arguments[1],1),r=this.length,e=S(t),i=d(e.length),o=0;if(i+n>r)throw V(Et);for(;o<i;)this[n+o]=e[o++]},Vt={entries:function(){return at.call(Nt(this))},keys:function(){return ft.call(Nt(this))},values:function(){return ct.call(Nt(this))}},zt=function(t,n){return w(t)&&t[_t]&&"symbol"!=typeof n&&n in t&&String(+n)==String(n)},qt=function(t,n){return zt(t,n=g(n,!0))?l(2,t[n]):B(t,n)},Kt=function(t,n,r){return!(zt(t,n=g(n,!0))&&w(r)&&b(r,"value"))||b(r,"get")||b(r,"set")||r.configurable||b(r,"writable")&&!r.writable||b(r,"enumerable")&&!r.enumerable?G(t,n,r):(t[n]=r.value,t)};St||(W.f=qt,U.f=Kt),u(u.S+u.F*!St,"Object",{getOwnPropertyDescriptor:qt,defineProperty:Kt}),o(function(){yt.call({})})&&(yt=gt=function(){return vt.call(this)});var Jt=v({},Wt);v(Jt,Vt),h(Jt,bt,Vt.values),v(Jt,{slice:Gt,set:Bt,constructor:function(){},toString:yt,toLocaleString:Ut}),Lt(Jt,"buffer","b"),Lt(Jt,"byteOffset","o"),Lt(Jt,"byteLength","l"),Lt(Jt,"length","e"),G(Jt,mt,{get:function(){return this[_t]}}),t.exports=function(t,n,r,f){f=!!f;var a=t+(f?"Clamped":"")+"Array",l="Uint8Array"!=a,v="get"+t,p="set"+t,y=i[a],g=y||{},b=y&&E(y),m=!y||!c.ABV,S={},_=y&&y[H],j=function(t,r){var e=t._d;return e.v[v](r*n+e.o,jt)},F=function(t,r,e){var i=t._d;f&&(e=(e=Math.round(e))<0?0:e>255?255:255&e),i.v[p](r*n+i.o,e,jt)},M=function(t,n){G(t,n,{get:function(){return j(this,n)},set:function(t){return F(this,n,t)},enumerable:!0})};m?(y=r(function(t,r,e,i){s(t,y,a,"_d");var o,u,c,f,l=0,v=0;if(w(r)){if(!(r instanceof X||(f=x(r))==K||f==J))return _t in r?kt(y,r):Rt.call(y,r);o=r,v=At(e,n);var p=r.byteLength;if(void 0===i){if(p%n)throw V(Et);if((u=p-v)<0)throw V(Et)}else if((u=d(i)*n)+v>p)throw V(Et);c=u/n}else c=Mt(r,!0),u=c*n,o=new X(u);for(h(t,"_d",{b:o,o:v,l:u,e:c,v:new Q(o)});l<c;)M(t,l++)}),_=y[H]=O(Jt),h(_,"constructor",y)):L(function(t){new y(null),new y(t)},!0)||(y=r(function(t,r,e,i){s(t,y,a);var o;return w(r)?r instanceof X||(o=x(r))==K||o==J?void 0!==i?new g(r,At(e,n),i):void 0!==e?new g(r,At(e,n)):new g(r):_t in r?kt(y,r):Rt.call(y,r):new g(Mt(r,l))}),Z(b!==Function.prototype?P(g).concat(P(b)):P(g),function(t){t in y||h(y,t,g[t])}),y[H]=_,e||(_.constructor=y));var A=_[bt],N=!!A&&("values"==A.name||void 0==A.name),T=Vt.values;h(y,xt,!0),h(_,_t,a),h(_,Ot,!0),h(_,wt,y),(f?new y(1)[mt]==a:mt in _)||G(_,mt,{get:function(){return a}}),S[a]=y,u(u.G+u.W+u.F*(y!=g),S),u(u.S,a,{BYTES_PER_ELEMENT:n,from:Rt,of:Ct}),Y in _||h(_,Y,n),u(u.P,a,Wt),R(a),u(u.P+u.F*Ft,a,{set:Bt}),u(u.P+u.F*!N,a,Vt),u(u.P+u.F*(_.toString!=yt),a,{toString:yt}),u(u.P+u.F*o(function(){new y(1).slice()}),a,{slice:Gt}),u(u.P+u.F*(o(function(){return[1,2].toLocaleString()!=new y([1,2]).toLocaleString()})||!o(function(){_.toLocaleString.call([1,2])})),a,{toLocaleString:Ut}),k[a]=N?A:T,e||N||h(_,bt,T)}}else t.exports=function(){}},function(t,n){var r={}.toString;t.exports=function(t){return r.call(t).slice(8,-1)}},function(t,n,r){var e=r(21),i=r(5).document,o=e(i)&&e(i.createElement);t.exports=function(t){return o?i.createElement(t):{}}},function(t,n,r){t.exports=!r(12)&&!r(18)(function(){return 7!=Object.defineProperty(r(57)("div"),"a",{get:function(){return 7}}).a})},function(t,n,r){"use strict";var e=r(36),i=r(51),o=r(64),u=r(13),c=r(8),f=r(35),a=r(96),s=r(38),l=r(103),h=r(15)("iterator"),v=!([].keys&&"next"in[].keys()),p="keys",d="values",y=function(){return this};t.exports=function(t,n,r,g,b,m,x){a(r,n,g);var w,S,_,O=function(t){if(!v&&t in F)return F[t];switch(t){case p:case d:return function(){return new r(this,t)}}return function(){return new r(this,t)}},E=n+" Iterator",P=b==d,j=!1,F=t.prototype,M=F[h]||F["@@iterator"]||b&&F[b],A=M||O(b),N=b?P?O("entries"):A:void 0,T="Array"==n?F.entries||M:M;if(T&&(_=l(T.call(new t)))!==Object.prototype&&(s(_,E,!0),e||c(_,h)||u(_,h,y)),P&&M&&M.name!==d&&(j=!0,A=function(){return M.call(this)}),e&&!x||!v&&!j&&F[h]||u(F,h,A),f[n]=A,f[E]=y,b)if(w={values:P?A:O(d),keys:m?A:O(p),entries:N},x)for(S in w)S in F||o(F,S,w[S]);else i(i.P+i.F*(v||j),n,w);return w}},function(t,n,r){var e=r(20),i=r(100),o=r(34),u=r(39)("IE_PROTO"),c=function(){},f="prototype",a=function(){var t,n=r(57)("iframe"),e=o.length;for(n.style.display="none",r(93).appendChild(n),n.src="javascript:",t=n.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),a=t.F;e--;)delete a[f][o[e]];return a()};t.exports=Object.create||function(t,n){var r;return null!==t?(c[f]=e(t),r=new c,c[f]=null,r[u]=t):r=a(),void 0===n?r:i(r,n)}},function(t,n,r){var e=r(63),i=r(34).concat("length","prototype");n.f=Object.getOwnPropertyNames||function(t){return e(t,i)}},function(t,n){n.f=Object.getOwnPropertySymbols},function(t,n,r){var e=r(8),i=r(9),o=r(90)(!1),u=r(39)("IE_PROTO");t.exports=function(t,n){var r,c=i(t),f=0,a=[];for(r in c)r!=u&&e(c,r)&&a.push(r);for(;n.length>f;)e(c,r=n[f++])&&(~o(a,r)||a.push(r));return a}},function(t,n,r){t.exports=r(13)},function(t,n,r){var e=r(76)("meta"),i=r(6),o=r(24),u=r(11).f,c=0,f=Object.isExtensible||function(){return!0},a=!r(4)(function(){return f(Object.preventExtensions({}))}),s=function(t){u(t,e,{value:{i:"O"+ ++c,w:{}}})},l=function(t,n){if(!i(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!o(t,e)){if(!f(t))return"F";if(!n)return"E";s(t)}return t[e].i},h=function(t,n){if(!o(t,e)){if(!f(t))return!0;if(!n)return!1;s(t)}return t[e].w},v=function(t){return a&&p.NEED&&f(t)&&!o(t,e)&&s(t),t},p=t.exports={KEY:e,NEED:!1,fastKey:l,getWeak:h,onFreeze:v}},function(t,n){t.exports=function(t,n){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:n}}},function(t,n){var r=Math.ceil,e=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?e:r)(t)}},function(t,n){t.exports=function(t,n,r,e){if(!(t instanceof n)||void 0!==e&&e in t)throw TypeError(r+": incorrect invocation!");return t}},function(t,n){t.exports=!1},function(t,n,r){var e=r(2),i=r(173),o=r(133),u=r(145)("IE_PROTO"),c=function(){},f="prototype",a=function(){var t,n=r(132)("iframe"),e=o.length;for(n.style.display="none",r(135).appendChild(n),n.src="javascript:",t=n.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),a=t.F;e--;)delete a[f][o[e]];return a()};t.exports=Object.create||function(t,n){var r;return null!==t?(c[f]=e(t),r=new c,c[f]=null,r[u]=t):r=a(),void 0===n?r:i(r,n)}},function(t,n,r){var e=r(175),i=r(133).concat("length","prototype");n.f=Object.getOwnPropertyNames||function(t){return e(t,i)}},function(t,n,r){var e=r(175),i=r(133);t.exports=Object.keys||function(t){return e(t,i)}},function(t,n,r){var e=r(28);t.exports=function(t,n,r){for(var i in n)e(t,i,n[i],r);return t}},function(t,n,r){"use strict";var e=r(3),i=r(11),o=r(10),u=r(7)("species");t.exports=function(t){var n=e[t];o&&n&&!n[u]&&i.f(n,u,{configurable:!0,get:function(){return this}})}},function(t,n,r){var e=r(67),i=Math.max,o=Math.min;t.exports=function(t,n){return t=e(t),t<0?i(t+n,0):o(t,n)}},function(t,n){var r=0,e=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++r+e).toString(36))}},function(t,n,r){var e=r(33);t.exports=function(t){return Object(e(t))}},function(t,n,r){var e=r(7)("unscopables"),i=Array.prototype;void 0==i[e]&&r(27)(i,e,{}),t.exports=function(t){i[e][t]=!0}},function(t,n,r){var e=r(53),i=r(169),o=r(137),u=r(2),c=r(16),f=r(154),a={},s={},n=t.exports=function(t,n,r,l,h){var v,p,d,y,g=h?function(){return t}:f(t),b=e(r,l,n?2:1),m=0;if("function"!=typeof g)throw TypeError(t+" is not iterable!");if(o(g)){for(v=c(t.length);v>m;m++)if((y=n?b(u(p=t[m])[0],p[1]):b(t[m]))===a||y===s)return y}else for(d=g.call(t);!(p=d.next()).done;)if((y=i(d,b,p.value,n))===a||y===s)return y};n.BREAK=a,n.RETURN=s},function(t,n){t.exports={}},function(t,n,r){var e=r(11).f,i=r(24),o=r(7)("toStringTag");t.exports=function(t,n,r){t&&!i(t=r?t:t.prototype,o)&&e(t,o,{configurable:!0,value:n})}},function(t,n,r){var e=r(1),i=r(46),o=r(4),u=r(150),c="["+u+"]",f="​",a=RegExp("^"+c+c+"*"),s=RegExp(c+c+"*$"),l=function(t,n,r){var i={},c=o(function(){return!!u[t]()||f[t]()!=f}),a=i[t]=c?n(h):u[t];r&&(i[r]=a),e(e.P+e.F*c,"String",i)},h=l.trim=function(t,n){return t=String(i(t)),1&n&&(t=t.replace(a,"")),2&n&&(t=t.replace(s,"")),t};t.exports=l},function(t,n,r){t.exports={default:r(86),__esModule:!0}},function(t,n,r){t.exports={default:r(87),__esModule:!0}},function(t,n,r){"use strict";function e(t){return t&&t.__esModule?t:{default:t}}n.__esModule=!0;var i=r(84),o=e(i),u=r(83),c=e(u),f="function"==typeof c.default&&"symbol"==typeof o.default?function(t){return typeof t}:function(t){return t&&"function"==typeof c.default&&t.constructor===c.default&&t!==c.default.prototype?"symbol":typeof t};n.default="function"==typeof c.default&&"symbol"===f(o.default)?function(t){return void 0===t?"undefined":f(t)}:function(t){return t&&"function"==typeof c.default&&t.constructor===c.default&&t!==c.default.prototype?"symbol":void 0===t?"undefined":f(t)}},function(t,n,r){r(110),r(108),r(111),r(112),t.exports=r(25).Symbol},function(t,n,r){r(109),r(113),t.exports=r(44).f("iterator")},function(t,n){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,n){t.exports=function(){}},function(t,n,r){var e=r(9),i=r(106),o=r(105);t.exports=function(t){return function(n,r,u){var c,f=e(n),a=i(f.length),s=o(u,a);if(t&&r!=r){for(;a>s;)if((c=f[s++])!=c)return!0}else for(;a>s;s++)if((t||s in f)&&f[s]===r)return t||s||0;return!t&&-1}}},function(t,n,r){var e=r(88);t.exports=function(t,n,r){if(e(t),void 0===n)return t;switch(r){case 1:return function(r){return t.call(n,r)};case 2:return function(r,e){return t.call(n,r,e)};case 3:return function(r,e,i){return t.call(n,r,e,i)}}return function(){return t.apply(n,arguments)}}},function(t,n,r){var e=r(19),i=r(62),o=r(37);t.exports=function(t){var n=e(t),r=i.f;if(r)for(var u,c=r(t),f=o.f,a=0;c.length>a;)f.call(t,u=c[a++])&&n.push(u);return n}},function(t,n,r){t.exports=r(5).document&&document.documentElement},function(t,n,r){var e=r(56);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==e(t)?t.split(""):Object(t)}},function(t,n,r){var e=r(56);t.exports=Array.isArray||function(t){return"Array"==e(t)}},function(t,n,r){"use strict";var e=r(60),i=r(22),o=r(38),u={};r(13)(u,r(15)("iterator"),function(){return this}),t.exports=function(t,n,r){t.prototype=e(u,{next:i(1,r)}),o(t,n+" Iterator")}},function(t,n){t.exports=function(t,n){return{value:n,done:!!t}}},function(t,n,r){var e=r(19),i=r(9);t.exports=function(t,n){for(var r,o=i(t),u=e(o),c=u.length,f=0;c>f;)if(o[r=u[f++]]===n)return r}},function(t,n,r){var e=r(23)("meta"),i=r(21),o=r(8),u=r(14).f,c=0,f=Object.isExtensible||function(){return!0},a=!r(18)(function(){return f(Object.preventExtensions({}))}),s=function(t){u(t,e,{value:{i:"O"+ ++c,w:{}}})},l=function(t,n){if(!i(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!o(t,e)){if(!f(t))return"F";if(!n)return"E";s(t)}return t[e].i},h=function(t,n){if(!o(t,e)){if(!f(t))return!0;if(!n)return!1;s(t)}return t[e].w},v=function(t){return a&&p.NEED&&f(t)&&!o(t,e)&&s(t),t},p=t.exports={KEY:e,NEED:!1,fastKey:l,getWeak:h,onFreeze:v}},function(t,n,r){var e=r(14),i=r(20),o=r(19);t.exports=r(12)?Object.defineProperties:function(t,n){i(t);for(var r,u=o(n),c=u.length,f=0;c>f;)e.f(t,r=u[f++],n[r]);return t}},function(t,n,r){var e=r(37),i=r(22),o=r(9),u=r(42),c=r(8),f=r(58),a=Object.getOwnPropertyDescriptor;n.f=r(12)?a:function(t,n){if(t=o(t),n=u(n,!0),f)try{return a(t,n)}catch(t){}if(c(t,n))return i(!e.f.call(t,n),t[n])}},function(t,n,r){var e=r(9),i=r(61).f,o={}.toString,u="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],c=function(t){try{return i(t)}catch(t){return u.slice()}};t.exports.f=function(t){return u&&"[object Window]"==o.call(t)?c(t):i(e(t))}},function(t,n,r){var e=r(8),i=r(77),o=r(39)("IE_PROTO"),u=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=i(t),e(t,o)?t[o]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?u:null}},function(t,n,r){var e=r(41),i=r(33);t.exports=function(t){return function(n,r){var o,u,c=String(i(n)),f=e(r),a=c.length;return f<0||f>=a?t?"":void 0:(o=c.charCodeAt(f),o<55296||o>56319||f+1===a||(u=c.charCodeAt(f+1))<56320||u>57343?t?c.charAt(f):o:t?c.slice(f,f+2):u-56320+(o-55296<<10)+65536)}}},function(t,n,r){var e=r(41),i=Math.max,o=Math.min;t.exports=function(t,n){return t=e(t),t<0?i(t+n,0):o(t,n)}},function(t,n,r){var e=r(41),i=Math.min;t.exports=function(t){return t>0?i(e(t),9007199254740991):0}},function(t,n,r){"use strict";var e=r(89),i=r(97),o=r(35),u=r(9);t.exports=r(59)(Array,"Array",function(t,n){this._t=u(t),this._i=0,this._k=n},function(){var t=this._t,n=this._k,r=this._i++;return!t||r>=t.length?(this._t=void 0,i(1)):"keys"==n?i(0,r):"values"==n?i(0,t[r]):i(0,[r,t[r]])},"values"),o.Arguments=o.Array,e("keys"),e("values"),e("entries")},function(t,n){},function(t,n,r){"use strict";var e=r(104)(!0);r(59)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,n=this._t,r=this._i;return r>=n.length?{value:void 0,done:!0}:(t=e(n,r),this._i+=t.length,{value:t,done:!1})})},function(t,n,r){"use strict";var e=r(5),i=r(8),o=r(12),u=r(51),c=r(64),f=r(99).KEY,a=r(18),s=r(40),l=r(38),h=r(23),v=r(15),p=r(44),d=r(43),y=r(98),g=r(92),b=r(95),m=r(20),x=r(9),w=r(42),S=r(22),_=r(60),O=r(102),E=r(101),P=r(14),j=r(19),F=E.f,M=P.f,A=O.f,N=e.Symbol,T=e.JSON,I=T&&T.stringify,k="prototype",L=v("_hidden"),R=v("toPrimitive"),C={}.propertyIsEnumerable,D=s("symbol-registry"),U=s("symbols"),W=s("op-symbols"),G=Object[k],B="function"==typeof N,V=e.QObject,z=!V||!V[k]||!V[k].findChild,q=o&&a(function(){return 7!=_(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,n,r){var e=F(G,n);e&&delete G[n],M(t,n,r),e&&t!==G&&M(G,n,e)}:M,K=function(t){var n=U[t]=_(N[k]);return n._k=t,n},J=B&&"symbol"==typeof N.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof N},Y=function(t,n,r){return t===G&&Y(W,n,r),m(t),n=w(n,!0),m(r),i(U,n)?(r.enumerable?(i(t,L)&&t[L][n]&&(t[L][n]=!1),r=_(r,{enumerable:S(0,!1)})):(i(t,L)||M(t,L,S(1,{})),t[L][n]=!0),q(t,n,r)):M(t,n,r)},H=function(t,n){m(t);for(var r,e=g(n=x(n)),i=0,o=e.length;o>i;)Y(t,r=e[i++],n[r]);return t},$=function(t,n){return void 0===n?_(t):H(_(t),n)},X=function(t){var n=C.call(this,t=w(t,!0));return!(this===G&&i(U,t)&&!i(W,t))&&(!(n||!i(this,t)||!i(U,t)||i(this,L)&&this[L][t])||n)},Q=function(t,n){if(t=x(t),n=w(n,!0),t!==G||!i(U,n)||i(W,n)){var r=F(t,n);return!r||!i(U,n)||i(t,L)&&t[L][n]||(r.enumerable=!0),r}},Z=function(t){for(var n,r=A(x(t)),e=[],o=0;r.length>o;)i(U,n=r[o++])||n==L||n==f||e.push(n);return e},tt=function(t){for(var n,r=t===G,e=A(r?W:x(t)),o=[],u=0;e.length>u;)!i(U,n=e[u++])||r&&!i(G,n)||o.push(U[n]);return o};B||(N=function(){if(this instanceof N)throw TypeError("Symbol is not a constructor!");var t=h(arguments.length>0?arguments[0]:void 0),n=function(r){this===G&&n.call(W,r),i(this,L)&&i(this[L],t)&&(this[L][t]=!1),q(this,t,S(1,r))};return o&&z&&q(G,t,{configurable:!0,set:n}),K(t)},c(N[k],"toString",function(){return this._k}),E.f=Q,P.f=Y,r(61).f=O.f=Z,r(37).f=X,r(62).f=tt,o&&!r(36)&&c(G,"propertyIsEnumerable",X,!0),p.f=function(t){return K(v(t))}),u(u.G+u.W+u.F*!B,{Symbol:N});for(var nt="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),rt=0;nt.length>rt;)v(nt[rt++]);for(var nt=j(v.store),rt=0;nt.length>rt;)d(nt[rt++]);u(u.S+u.F*!B,"Symbol",{for:function(t){return i(D,t+="")?D[t]:D[t]=N(t)},keyFor:function(t){if(J(t))return y(D,t);throw TypeError(t+" is not a symbol!")},useSetter:function(){z=!0},useSimple:function(){z=!1}}),u(u.S+u.F*!B,"Object",{create:$,defineProperty:Y,defineProperties:H,getOwnPropertyDescriptor:Q,getOwnPropertyNames:Z,getOwnPropertySymbols:tt}),T&&u(u.S+u.F*(!B||a(function(){var t=N();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!J(t)){for(var n,r,e=[t],i=1;arguments.length>i;)e.push(arguments[i++]);return n=e[1],"function"==typeof n&&(r=n),!r&&b(n)||(n=function(t,n){if(r&&(n=r.call(this,t,n)),!J(n))return n}),e[1]=n,I.apply(T,e)}}}),N[k][R]||r(13)(N[k],R,N[k].valueOf),l(N,"Symbol"),l(Math,"Math",!0),l(e.JSON,"JSON",!0)},function(t,n,r){r(43)("asyncIterator")},function(t,n,r){r(43)("observable")},function(t,n,r){r(107);for(var e=r(5),i=r(13),o=r(35),u=r(15)("toStringTag"),c=["NodeList","DOMTokenList","MediaList","StyleSheetList","CSSRuleList"],f=0;f<5;f++){var a=c[f],s=e[a],l=s&&s.prototype;l&&!l[u]&&i(l,u,a),o[a]=o.Array}},function(t,n,r){var e=r(45),i=r(7)("toStringTag"),o="Arguments"==e(function(){return arguments}()),u=function(t,n){try{return t[n]}catch(t){}};t.exports=function(t){var n,r,c;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(r=u(n=Object(t),i))?r:o?e(n):"Object"==(c=e(n))&&"function"==typeof n.callee?"Arguments":c}},function(t,n,r){var e=r(45);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==e(t)?t.split(""):Object(t)}},function(t,n){n.f={}.propertyIsEnumerable},function(t,n,r){var e=r(30),i=r(16),o=r(75);t.exports=function(t){return function(n,r,u){var c,f=e(n),a=i(f.length),s=o(u,a);if(t&&r!=r){for(;a>s;)if((c=f[s++])!=c)return!0}else for(;a>s;s++)if((t||s in f)&&f[s]===r)return t||s||0;return!t&&-1}}},function(t,n,r){"use strict";var e=r(3),i=r(1),o=r(28),u=r(73),c=r(65),f=r(79),a=r(68),s=r(6),l=r(4),h=r(123),v=r(81),p=r(136);t.exports=function(t,n,r,d,y,g){var b=e[t],m=b,x=y?"set":"add",w=m&&m.prototype,S={},_=function(t){var n=w[t];o(w,t,"delete"==t?function(t){return!(g&&!s(t))&&n.call(this,0===t?0:t)}:"has"==t?function(t){return!(g&&!s(t))&&n.call(this,0===t?0:t)}:"get"==t?function(t){return g&&!s(t)?void 0:n.call(this,0===t?0:t)}:"add"==t?function(t){return n.call(this,0===t?0:t),this}:function(t,r){return n.call(this,0===t?0:t,r),this})};if("function"==typeof m&&(g||w.forEach&&!l(function(){(new m).entries().next()}))){var O=new m,E=O[x](g?{}:-0,1)!=O,P=l(function(){O.has(1)}),j=h(function(t){new m(t)}),F=!g&&l(function(){for(var t=new m,n=5;n--;)t[x](n,n);return!t.has(-0)});j||(m=n(function(n,r){a(n,m,t);var e=p(new b,n,m);return void 0!=r&&f(r,y,e[x],e),e}),m.prototype=w,w.constructor=m),(P||F)&&(_("delete"),_("has"),y&&_("get")),(F||E)&&_(x),g&&w.clear&&delete w.clear}else m=d.getConstructor(n,t,y,x),u(m.prototype,r),c.NEED=!0;return v(m,t),S[t]=m,i(i.G+i.W+i.F*(m!=b),S),g||d.setStrong(m,t,y),m}},function(t,n,r){"use strict";var e=r(27),i=r(28),o=r(4),u=r(46),c=r(7);t.exports=function(t,n,r){var f=c(t),a=r(u,f,""[t]),s=a[0],l=a[1];o(function(){var n={};return n[f]=function(){return 7},7!=""[t](n)})&&(i(String.prototype,t,s),e(RegExp.prototype,f,2==n?function(t,n){return l.call(t,this,n)}:function(t){return l.call(t,this)}))}
},function(t,n,r){"use strict";var e=r(2);t.exports=function(){var t=e(this),n="";return t.global&&(n+="g"),t.ignoreCase&&(n+="i"),t.multiline&&(n+="m"),t.unicode&&(n+="u"),t.sticky&&(n+="y"),n}},function(t,n){t.exports=function(t,n,r){var e=void 0===r;switch(n.length){case 0:return e?t():t.call(r);case 1:return e?t(n[0]):t.call(r,n[0]);case 2:return e?t(n[0],n[1]):t.call(r,n[0],n[1]);case 3:return e?t(n[0],n[1],n[2]):t.call(r,n[0],n[1],n[2]);case 4:return e?t(n[0],n[1],n[2],n[3]):t.call(r,n[0],n[1],n[2],n[3])}return t.apply(r,n)}},function(t,n,r){var e=r(6),i=r(45),o=r(7)("match");t.exports=function(t){var n;return e(t)&&(void 0!==(n=t[o])?!!n:"RegExp"==i(t))}},function(t,n,r){var e=r(7)("iterator"),i=!1;try{var o=[7][e]();o.return=function(){i=!0},Array.from(o,function(){throw 2})}catch(t){}t.exports=function(t,n){if(!n&&!i)return!1;var r=!1;try{var o=[7],u=o[e]();u.next=function(){return{done:r=!0}},o[e]=function(){return u},t(o)}catch(t){}return r}},function(t,n,r){t.exports=r(69)||!r(4)(function(){var t=Math.random();__defineSetter__.call(null,t,function(){}),delete r(3)[t]})},function(t,n){n.f=Object.getOwnPropertySymbols},function(t,n,r){var e=r(3),i="__core-js_shared__",o=e[i]||(e[i]={});t.exports=function(t){return o[t]||(o[t]={})}},function(t,n,r){for(var e,i=r(3),o=r(27),u=r(76),c=u("typed_array"),f=u("view"),a=!(!i.ArrayBuffer||!i.DataView),s=a,l=0,h="Int8Array,Uint8Array,Uint8ClampedArray,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array".split(",");l<9;)(e=i[h[l++]])?(o(e.prototype,c,!0),o(e.prototype,f,!0)):s=!1;t.exports={ABV:a,CONSTR:s,TYPED:c,VIEW:f}},function(t,n){"use strict";var r={versions:function(){var t=window.navigator.userAgent;return{trident:t.indexOf("Trident")>-1,presto:t.indexOf("Presto")>-1,webKit:t.indexOf("AppleWebKit")>-1,gecko:t.indexOf("Gecko")>-1&&-1==t.indexOf("KHTML"),mobile:!!t.match(/AppleWebKit.*Mobile.*/),ios:!!t.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/),android:t.indexOf("Android")>-1||t.indexOf("Linux")>-1,iPhone:t.indexOf("iPhone")>-1||t.indexOf("Mac")>-1,iPad:t.indexOf("iPad")>-1,webApp:-1==t.indexOf("Safari"),weixin:-1==t.indexOf("MicroMessenger")}}()};t.exports=r},function(t,n,r){"use strict";var e=r(85),i=function(t){return t&&t.__esModule?t:{default:t}}(e),o=function(){function t(t,n,e){return n||e?String.fromCharCode(n||e):r[t]||t}function n(t){return e[t]}var r={"&quot;":'"',"&lt;":"<","&gt;":">","&amp;":"&","&nbsp;":" "},e={};for(var u in r)e[r[u]]=u;return r["&apos;"]="'",e["'"]="&#39;",{encode:function(t){return t?(""+t).replace(/['<> "&]/g,n).replace(/\r?\n/g,"<br/>").replace(/\s/g,"&nbsp;"):""},decode:function(n){return n?(""+n).replace(/<br\s*\/?>/gi,"\n").replace(/&quot;|&lt;|&gt;|&amp;|&nbsp;|&apos;|&#(\d+);|&#(\d+)/g,t).replace(/\u00a0/g," "):""},encodeBase16:function(t){if(!t)return t;t+="";for(var n=[],r=0,e=t.length;e>r;r++)n.push(t.charCodeAt(r).toString(16).toUpperCase());return n.join("")},encodeBase16forJSON:function(t){if(!t)return t;t=t.replace(/[\u4E00-\u9FBF]/gi,function(t){return escape(t).replace("%u","\\u")});for(var n=[],r=0,e=t.length;e>r;r++)n.push(t.charCodeAt(r).toString(16).toUpperCase());return n.join("")},decodeBase16:function(t){if(!t)return t;t+="";for(var n=[],r=0,e=t.length;e>r;r+=2)n.push(String.fromCharCode("0x"+t.slice(r,r+2)));return n.join("")},encodeObject:function(t){if(t instanceof Array)for(var n=0,r=t.length;r>n;n++)t[n]=o.encodeObject(t[n]);else if("object"==(void 0===t?"undefined":(0,i.default)(t)))for(var e in t)t[e]=o.encodeObject(t[e]);else if("string"==typeof t)return o.encode(t);return t},loadScript:function(t){var n=document.createElement("script");document.getElementsByTagName("body")[0].appendChild(n),n.setAttribute("src",t)},addLoadEvent:function(t){var n=window.onload;"function"!=typeof window.onload?window.onload=t:window.onload=function(){n(),t()}}}}();t.exports=o},function(t,n,r){"use strict";var e=r(17),i=r(75),o=r(16);t.exports=function(t){for(var n=e(this),r=o(n.length),u=arguments.length,c=i(u>1?arguments[1]:void 0,r),f=u>2?arguments[2]:void 0,a=void 0===f?r:i(f,r);a>c;)n[c++]=t;return n}},function(t,n,r){"use strict";var e=r(11),i=r(66);t.exports=function(t,n,r){n in t?e.f(t,n,i(0,r)):t[n]=r}},function(t,n,r){var e=r(6),i=r(3).document,o=e(i)&&e(i.createElement);t.exports=function(t){return o?i.createElement(t):{}}},function(t,n){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,n,r){var e=r(7)("match");t.exports=function(t){var n=/./;try{"/./"[t](n)}catch(r){try{return n[e]=!1,!"/./"[t](n)}catch(t){}}return!0}},function(t,n,r){t.exports=r(3).document&&document.documentElement},function(t,n,r){var e=r(6),i=r(144).set;t.exports=function(t,n,r){var o,u=n.constructor;return u!==r&&"function"==typeof u&&(o=u.prototype)!==r.prototype&&e(o)&&i&&i(t,o),t}},function(t,n,r){var e=r(80),i=r(7)("iterator"),o=Array.prototype;t.exports=function(t){return void 0!==t&&(e.Array===t||o[i]===t)}},function(t,n,r){var e=r(45);t.exports=Array.isArray||function(t){return"Array"==e(t)}},function(t,n,r){"use strict";var e=r(70),i=r(66),o=r(81),u={};r(27)(u,r(7)("iterator"),function(){return this}),t.exports=function(t,n,r){t.prototype=e(u,{next:i(1,r)}),o(t,n+" Iterator")}},function(t,n,r){"use strict";var e=r(69),i=r(1),o=r(28),u=r(27),c=r(24),f=r(80),a=r(139),s=r(81),l=r(32),h=r(7)("iterator"),v=!([].keys&&"next"in[].keys()),p="keys",d="values",y=function(){return this};t.exports=function(t,n,r,g,b,m,x){a(r,n,g);var w,S,_,O=function(t){if(!v&&t in F)return F[t];switch(t){case p:case d:return function(){return new r(this,t)}}return function(){return new r(this,t)}},E=n+" Iterator",P=b==d,j=!1,F=t.prototype,M=F[h]||F["@@iterator"]||b&&F[b],A=M||O(b),N=b?P?O("entries"):A:void 0,T="Array"==n?F.entries||M:M;if(T&&(_=l(T.call(new t)))!==Object.prototype&&(s(_,E,!0),e||c(_,h)||u(_,h,y)),P&&M&&M.name!==d&&(j=!0,A=function(){return M.call(this)}),e&&!x||!v&&!j&&F[h]||u(F,h,A),f[n]=A,f[E]=y,b)if(w={values:P?A:O(d),keys:m?A:O(p),entries:N},x)for(S in w)S in F||o(F,S,w[S]);else i(i.P+i.F*(v||j),n,w);return w}},function(t,n){var r=Math.expm1;t.exports=!r||r(10)>22025.465794806718||r(10)<22025.465794806718||-2e-17!=r(-2e-17)?function(t){return 0==(t=+t)?t:t>-1e-6&&t<1e-6?t+t*t/2:Math.exp(t)-1}:r},function(t,n){t.exports=Math.sign||function(t){return 0==(t=+t)||t!=t?t:t<0?-1:1}},function(t,n,r){var e=r(3),i=r(151).set,o=e.MutationObserver||e.WebKitMutationObserver,u=e.process,c=e.Promise,f="process"==r(45)(u);t.exports=function(){var t,n,r,a=function(){var e,i;for(f&&(e=u.domain)&&e.exit();t;){i=t.fn,t=t.next;try{i()}catch(e){throw t?r():n=void 0,e}}n=void 0,e&&e.enter()};if(f)r=function(){u.nextTick(a)};else if(o){var s=!0,l=document.createTextNode("");new o(a).observe(l,{characterData:!0}),r=function(){l.data=s=!s}}else if(c&&c.resolve){var h=c.resolve();r=function(){h.then(a)}}else r=function(){i.call(e,a)};return function(e){var i={fn:e,next:void 0};n&&(n.next=i),t||(t=i,r()),n=i}}},function(t,n,r){var e=r(6),i=r(2),o=function(t,n){if(i(t),!e(n)&&null!==n)throw TypeError(n+": can't set as prototype!")};t.exports={set:Object.setPrototypeOf||("__proto__"in{}?function(t,n,e){try{e=r(53)(Function.call,r(31).f(Object.prototype,"__proto__").set,2),e(t,[]),n=!(t instanceof Array)}catch(t){n=!0}return function(t,r){return o(t,r),n?t.__proto__=r:e(t,r),t}}({},!1):void 0),check:o}},function(t,n,r){var e=r(126)("keys"),i=r(76);t.exports=function(t){return e[t]||(e[t]=i(t))}},function(t,n,r){var e=r(2),i=r(26),o=r(7)("species");t.exports=function(t,n){var r,u=e(t).constructor;return void 0===u||void 0==(r=e(u)[o])?n:i(r)}},function(t,n,r){var e=r(67),i=r(46);t.exports=function(t){return function(n,r){var o,u,c=String(i(n)),f=e(r),a=c.length;return f<0||f>=a?t?"":void 0:(o=c.charCodeAt(f),o<55296||o>56319||f+1===a||(u=c.charCodeAt(f+1))<56320||u>57343?t?c.charAt(f):o:t?c.slice(f,f+2):u-56320+(o-55296<<10)+65536)}}},function(t,n,r){var e=r(122),i=r(46);t.exports=function(t,n,r){if(e(n))throw TypeError("String#"+r+" doesn't accept regex!");return String(i(t))}},function(t,n,r){"use strict";var e=r(67),i=r(46);t.exports=function(t){var n=String(i(this)),r="",o=e(t);if(o<0||o==1/0)throw RangeError("Count can't be negative");for(;o>0;(o>>>=1)&&(n+=n))1&o&&(r+=n);return r}},function(t,n){t.exports="\t\n\v\f\r   ᠎             　\u2028\u2029\ufeff"},function(t,n,r){var e,i,o,u=r(53),c=r(121),f=r(135),a=r(132),s=r(3),l=s.process,h=s.setImmediate,v=s.clearImmediate,p=s.MessageChannel,d=0,y={},g="onreadystatechange",b=function(){var t=+this;if(y.hasOwnProperty(t)){var n=y[t];delete y[t],n()}},m=function(t){b.call(t.data)};h&&v||(h=function(t){for(var n=[],r=1;arguments.length>r;)n.push(arguments[r++]);return y[++d]=function(){c("function"==typeof t?t:Function(t),n)},e(d),d},v=function(t){delete y[t]},"process"==r(45)(l)?e=function(t){l.nextTick(u(b,t,1))}:p?(i=new p,o=i.port2,i.port1.onmessage=m,e=u(o.postMessage,o,1)):s.addEventListener&&"function"==typeof postMessage&&!s.importScripts?(e=function(t){s.postMessage(t+"","*")},s.addEventListener("message",m,!1)):e=g in a("script")?function(t){f.appendChild(a("script"))[g]=function(){f.removeChild(this),b.call(t)}}:function(t){setTimeout(u(b,t,1),0)}),t.exports={set:h,clear:v}},function(t,n,r){"use strict";var e=r(3),i=r(10),o=r(69),u=r(127),c=r(27),f=r(73),a=r(4),s=r(68),l=r(67),h=r(16),v=r(71).f,p=r(11).f,d=r(130),y=r(81),g="ArrayBuffer",b="DataView",m="prototype",x="Wrong length!",w="Wrong index!",S=e[g],_=e[b],O=e.Math,E=e.RangeError,P=e.Infinity,j=S,F=O.abs,M=O.pow,A=O.floor,N=O.log,T=O.LN2,I="buffer",k="byteLength",L="byteOffset",R=i?"_b":I,C=i?"_l":k,D=i?"_o":L,U=function(t,n,r){var e,i,o,u=Array(r),c=8*r-n-1,f=(1<<c)-1,a=f>>1,s=23===n?M(2,-24)-M(2,-77):0,l=0,h=t<0||0===t&&1/t<0?1:0;for(t=F(t),t!=t||t===P?(i=t!=t?1:0,e=f):(e=A(N(t)/T),t*(o=M(2,-e))<1&&(e--,o*=2),t+=e+a>=1?s/o:s*M(2,1-a),t*o>=2&&(e++,o/=2),e+a>=f?(i=0,e=f):e+a>=1?(i=(t*o-1)*M(2,n),e+=a):(i=t*M(2,a-1)*M(2,n),e=0));n>=8;u[l++]=255&i,i/=256,n-=8);for(e=e<<n|i,c+=n;c>0;u[l++]=255&e,e/=256,c-=8);return u[--l]|=128*h,u},W=function(t,n,r){var e,i=8*r-n-1,o=(1<<i)-1,u=o>>1,c=i-7,f=r-1,a=t[f--],s=127&a;for(a>>=7;c>0;s=256*s+t[f],f--,c-=8);for(e=s&(1<<-c)-1,s>>=-c,c+=n;c>0;e=256*e+t[f],f--,c-=8);if(0===s)s=1-u;else{if(s===o)return e?NaN:a?-P:P;e+=M(2,n),s-=u}return(a?-1:1)*e*M(2,s-n)},G=function(t){return t[3]<<24|t[2]<<16|t[1]<<8|t[0]},B=function(t){return[255&t]},V=function(t){return[255&t,t>>8&255]},z=function(t){return[255&t,t>>8&255,t>>16&255,t>>24&255]},q=function(t){return U(t,52,8)},K=function(t){return U(t,23,4)},J=function(t,n,r){p(t[m],n,{get:function(){return this[r]}})},Y=function(t,n,r,e){var i=+r,o=l(i);if(i!=o||o<0||o+n>t[C])throw E(w);var u=t[R]._b,c=o+t[D],f=u.slice(c,c+n);return e?f:f.reverse()},H=function(t,n,r,e,i,o){var u=+r,c=l(u);if(u!=c||c<0||c+n>t[C])throw E(w);for(var f=t[R]._b,a=c+t[D],s=e(+i),h=0;h<n;h++)f[a+h]=s[o?h:n-h-1]},$=function(t,n){s(t,S,g);var r=+n,e=h(r);if(r!=e)throw E(x);return e};if(u.ABV){if(!a(function(){new S})||!a(function(){new S(.5)})){S=function(t){return new j($(this,t))};for(var X,Q=S[m]=j[m],Z=v(j),tt=0;Z.length>tt;)(X=Z[tt++])in S||c(S,X,j[X]);o||(Q.constructor=S)}var nt=new _(new S(2)),rt=_[m].setInt8;nt.setInt8(0,2147483648),nt.setInt8(1,2147483649),!nt.getInt8(0)&&nt.getInt8(1)||f(_[m],{setInt8:function(t,n){rt.call(this,t,n<<24>>24)},setUint8:function(t,n){rt.call(this,t,n<<24>>24)}},!0)}else S=function(t){var n=$(this,t);this._b=d.call(Array(n),0),this[C]=n},_=function(t,n,r){s(this,_,b),s(t,S,b);var e=t[C],i=l(n);if(i<0||i>e)throw E("Wrong offset!");if(r=void 0===r?e-i:h(r),i+r>e)throw E(x);this[R]=t,this[D]=i,this[C]=r},i&&(J(S,k,"_l"),J(_,I,"_b"),J(_,k,"_l"),J(_,L,"_o")),f(_[m],{getInt8:function(t){return Y(this,1,t)[0]<<24>>24},getUint8:function(t){return Y(this,1,t)[0]},getInt16:function(t){var n=Y(this,2,t,arguments[1]);return(n[1]<<8|n[0])<<16>>16},getUint16:function(t){var n=Y(this,2,t,arguments[1]);return n[1]<<8|n[0]},getInt32:function(t){return G(Y(this,4,t,arguments[1]))},getUint32:function(t){return G(Y(this,4,t,arguments[1]))>>>0},getFloat32:function(t){return W(Y(this,4,t,arguments[1]),23,4)},getFloat64:function(t){return W(Y(this,8,t,arguments[1]),52,8)},setInt8:function(t,n){H(this,1,t,B,n)},setUint8:function(t,n){H(this,1,t,B,n)},setInt16:function(t,n){H(this,2,t,V,n,arguments[2])},setUint16:function(t,n){H(this,2,t,V,n,arguments[2])},setInt32:function(t,n){H(this,4,t,z,n,arguments[2])},setUint32:function(t,n){H(this,4,t,z,n,arguments[2])},setFloat32:function(t,n){H(this,4,t,K,n,arguments[2])},setFloat64:function(t,n){H(this,8,t,q,n,arguments[2])}});y(S,g),y(_,b),c(_[m],u.VIEW,!0),n[g]=S,n[b]=_},function(t,n,r){var e=r(3),i=r(52),o=r(69),u=r(182),c=r(11).f;t.exports=function(t){var n=i.Symbol||(i.Symbol=o?{}:e.Symbol||{});"_"==t.charAt(0)||t in n||c(n,t,{value:u.f(t)})}},function(t,n,r){var e=r(114),i=r(7)("iterator"),o=r(80);t.exports=r(52).getIteratorMethod=function(t){if(void 0!=t)return t[i]||t["@@iterator"]||o[e(t)]}},function(t,n,r){"use strict";var e=r(78),i=r(170),o=r(80),u=r(30);t.exports=r(140)(Array,"Array",function(t,n){this._t=u(t),this._i=0,this._k=n},function(){var t=this._t,n=this._k,r=this._i++;return!t||r>=t.length?(this._t=void 0,i(1)):"keys"==n?i(0,r):"values"==n?i(0,t[r]):i(0,[r,t[r]])},"values"),o.Arguments=o.Array,e("keys"),e("values"),e("entries")},function(t,n){function r(t,n){t.classList?t.classList.add(n):t.className+=" "+n}t.exports=r},function(t,n){function r(t,n){if(t.classList)t.classList.remove(n);else{var r=new RegExp("(^|\\b)"+n.split(" ").join("|")+"(\\b|$)","gi");t.className=t.className.replace(r," ")}}t.exports=r},function(t,n){function r(){throw new Error("setTimeout has not been defined")}function e(){throw new Error("clearTimeout has not been defined")}function i(t){if(s===setTimeout)return setTimeout(t,0);if((s===r||!s)&&setTimeout)return s=setTimeout,setTimeout(t,0);try{return s(t,0)}catch(n){try{return s.call(null,t,0)}catch(n){return s.call(this,t,0)}}}function o(t){if(l===clearTimeout)return clearTimeout(t);if((l===e||!l)&&clearTimeout)return l=clearTimeout,clearTimeout(t);try{return l(t)}catch(n){try{return l.call(null,t)}catch(n){return l.call(this,t)}}}function u(){d&&v&&(d=!1,v.length?p=v.concat(p):y=-1,p.length&&c())}function c(){if(!d){var t=i(u);d=!0;for(var n=p.length;n;){for(v=p,p=[];++y<n;)v&&v[y].run();y=-1,n=p.length}v=null,d=!1,o(t)}}function f(t,n){this.fun=t,this.array=n}function a(){}var s,l,h=t.exports={};!function(){try{s="function"==typeof setTimeout?setTimeout:r}catch(t){s=r}try{l="function"==typeof clearTimeout?clearTimeout:e}catch(t){l=e}}();var v,p=[],d=!1,y=-1;h.nextTick=function(t){var n=new Array(arguments.length-1);if(arguments.length>1)for(var r=1;r<arguments.length;r++)n[r-1]=arguments[r];p.push(new f(t,n)),1!==p.length||d||i(c)},f.prototype.run=function(){this.fun.apply(null,this.array)},h.title="browser",h.browser=!0,h.env={},h.argv=[],h.version="",h.versions={},h.on=a,h.addListener=a,h.once=a,h.off=a,h.removeListener=a,h.removeAllListeners=a,h.emit=a,h.prependListener=a,h.prependOnceListener=a,h.listeners=function(t){return[]},h.binding=function(t){throw new Error("process.binding is not supported")},h.cwd=function(){return"/"},h.chdir=function(t){throw new Error("process.chdir is not supported")},h.umask=function(){return 0}},function(t,n,r){var e=r(45);t.exports=function(t,n){if("number"!=typeof t&&"Number"!=e(t))throw TypeError(n);return+t}},function(t,n,r){"use strict";var e=r(17),i=r(75),o=r(16);t.exports=[].copyWithin||function(t,n){var r=e(this),u=o(r.length),c=i(t,u),f=i(n,u),a=arguments.length>2?arguments[2]:void 0,s=Math.min((void 0===a?u:i(a,u))-f,u-c),l=1;for(f<c&&c<f+s&&(l=-1,f+=s-1,c+=s-1);s-- >0;)f in r?r[c]=r[f]:delete r[c],c+=l,f+=l;return r}},function(t,n,r){var e=r(79);t.exports=function(t,n){var r=[];return e(t,!1,r.push,r,n),r}},function(t,n,r){var e=r(26),i=r(17),o=r(115),u=r(16);t.exports=function(t,n,r,c,f){e(n);var a=i(t),s=o(a),l=u(a.length),h=f?l-1:0,v=f?-1:1;if(r<2)for(;;){if(h in s){c=s[h],h+=v;break}if(h+=v,f?h<0:l<=h)throw TypeError("Reduce of empty array with no initial value")}for(;f?h>=0:l>h;h+=v)h in s&&(c=n(c,s[h],h,a));return c}},function(t,n,r){"use strict";var e=r(26),i=r(6),o=r(121),u=[].slice,c={},f=function(t,n,r){if(!(n in c)){for(var e=[],i=0;i<n;i++)e[i]="a["+i+"]";c[n]=Function("F,a","return new F("+e.join(",")+")")}return c[n](t,r)};t.exports=Function.bind||function(t){var n=e(this),r=u.call(arguments,1),c=function(){var e=r.concat(u.call(arguments));return this instanceof c?f(n,e.length,e):o(n,e,t)};return i(n.prototype)&&(c.prototype=n.prototype),c}},function(t,n,r){"use strict";var e=r(11).f,i=r(70),o=r(73),u=r(53),c=r(68),f=r(46),a=r(79),s=r(140),l=r(170),h=r(74),v=r(10),p=r(65).fastKey,d=v?"_s":"size",y=function(t,n){var r,e=p(n);if("F"!==e)return t._i[e];for(r=t._f;r;r=r.n)if(r.k==n)return r};t.exports={getConstructor:function(t,n,r,s){var l=t(function(t,e){c(t,l,n,"_i"),t._i=i(null),t._f=void 0,t._l=void 0,t[d]=0,void 0!=e&&a(e,r,t[s],t)});return o(l.prototype,{clear:function(){for(var t=this,n=t._i,r=t._f;r;r=r.n)r.r=!0,r.p&&(r.p=r.p.n=void 0),delete n[r.i];t._f=t._l=void 0,t[d]=0},delete:function(t){var n=this,r=y(n,t);if(r){var e=r.n,i=r.p;delete n._i[r.i],r.r=!0,i&&(i.n=e),e&&(e.p=i),n._f==r&&(n._f=e),n._l==r&&(n._l=i),n[d]--}return!!r},forEach:function(t){c(this,l,"forEach");for(var n,r=u(t,arguments.length>1?arguments[1]:void 0,3);n=n?n.n:this._f;)for(r(n.v,n.k,this);n&&n.r;)n=n.p},has:function(t){return!!y(this,t)}}),v&&e(l.prototype,"size",{get:function(){return f(this[d])}}),l},def:function(t,n,r){var e,i,o=y(t,n);return o?o.v=r:(t._l=o={i:i=p(n,!0),k:n,v:r,p:e=t._l,n:void 0,r:!1},t._f||(t._f=o),e&&(e.n=o),t[d]++,"F"!==i&&(t._i[i]=o)),t},getEntry:y,setStrong:function(t,n,r){s(t,n,function(t,n){this._t=t,this._k=n,this._l=void 0},function(){for(var t=this,n=t._k,r=t._l;r&&r.r;)r=r.p;return t._t&&(t._l=r=r?r.n:t._t._f)?"keys"==n?l(0,r.k):"values"==n?l(0,r.v):l(0,[r.k,r.v]):(t._t=void 0,l(1))},r?"entries":"values",!r,!0),h(n)}}},function(t,n,r){var e=r(114),i=r(161);t.exports=function(t){return function(){if(e(this)!=t)throw TypeError(t+"#toJSON isn't generic");return i(this)}}},function(t,n,r){"use strict";var e=r(73),i=r(65).getWeak,o=r(2),u=r(6),c=r(68),f=r(79),a=r(48),s=r(24),l=a(5),h=a(6),v=0,p=function(t){return t._l||(t._l=new d)},d=function(){this.a=[]},y=function(t,n){return l(t.a,function(t){return t[0]===n})};d.prototype={get:function(t){var n=y(this,t);if(n)return n[1]},has:function(t){return!!y(this,t)},set:function(t,n){var r=y(this,t);r?r[1]=n:this.a.push([t,n])},delete:function(t){var n=h(this.a,function(n){return n[0]===t});return~n&&this.a.splice(n,1),!!~n}},t.exports={getConstructor:function(t,n,r,o){var a=t(function(t,e){c(t,a,n,"_i"),t._i=v++,t._l=void 0,void 0!=e&&f(e,r,t[o],t)});return e(a.prototype,{delete:function(t){if(!u(t))return!1;var n=i(t);return!0===n?p(this).delete(t):n&&s(n,this._i)&&delete n[this._i]},has:function(t){if(!u(t))return!1;var n=i(t);return!0===n?p(this).has(t):n&&s(n,this._i)}}),a},def:function(t,n,r){var e=i(o(n),!0);return!0===e?p(t).set(n,r):e[t._i]=r,t},ufstore:p}},function(t,n,r){t.exports=!r(10)&&!r(4)(function(){return 7!=Object.defineProperty(r(132)("div"),"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(6),i=Math.floor;t.exports=function(t){return!e(t)&&isFinite(t)&&i(t)===t}},function(t,n,r){var e=r(2);t.exports=function(t,n,r,i){try{return i?n(e(r)[0],r[1]):n(r)}catch(n){var o=t.return;throw void 0!==o&&e(o.call(t)),n}}},function(t,n){t.exports=function(t,n){return{value:n,done:!!t}}},function(t,n){t.exports=Math.log1p||function(t){return(t=+t)>-1e-8&&t<1e-8?t-t*t/2:Math.log(1+t)}},function(t,n,r){"use strict";var e=r(72),i=r(125),o=r(116),u=r(17),c=r(115),f=Object.assign;t.exports=!f||r(4)(function(){var t={},n={},r=Symbol(),e="abcdefghijklmnopqrst";return t[r]=7,e.split("").forEach(function(t){n[t]=t}),7!=f({},t)[r]||Object.keys(f({},n)).join("")!=e})?function(t,n){for(var r=u(t),f=arguments.length,a=1,s=i.f,l=o.f;f>a;)for(var h,v=c(arguments[a++]),p=s?e(v).concat(s(v)):e(v),d=p.length,y=0;d>y;)l.call(v,h=p[y++])&&(r[h]=v[h]);return r}:f},function(t,n,r){var e=r(11),i=r(2),o=r(72);t.exports=r(10)?Object.defineProperties:function(t,n){i(t);for(var r,u=o(n),c=u.length,f=0;c>f;)e.f(t,r=u[f++],n[r]);return t}},function(t,n,r){var e=r(30),i=r(71).f,o={}.toString,u="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],c=function(t){try{return i(t)}catch(t){return u.slice()}};t.exports.f=function(t){return u&&"[object Window]"==o.call(t)?c(t):i(e(t))}},function(t,n,r){var e=r(24),i=r(30),o=r(117)(!1),u=r(145)("IE_PROTO");t.exports=function(t,n){var r,c=i(t),f=0,a=[];for(r in c)r!=u&&e(c,r)&&a.push(r);for(;n.length>f;)e(c,r=n[f++])&&(~o(a,r)||a.push(r));return a}},function(t,n,r){var e=r(72),i=r(30),o=r(116).f;t.exports=function(t){return function(n){for(var r,u=i(n),c=e(u),f=c.length,a=0,s=[];f>a;)o.call(u,r=c[a++])&&s.push(t?[r,u[r]]:u[r]);return s}}},function(t,n,r){var e=r(71),i=r(125),o=r(2),u=r(3).Reflect;t.exports=u&&u.ownKeys||function(t){var n=e.f(o(t)),r=i.f;return r?n.concat(r(t)):n}},function(t,n,r){var e=r(3).parseFloat,i=r(82).trim;t.exports=1/e(r(150)+"-0")!=-1/0?function(t){var n=i(String(t),3),r=e(n);return 0===r&&"-"==n.charAt(0)?-0:r}:e},function(t,n,r){var e=r(3).parseInt,i=r(82).trim,o=r(150),u=/^[\-+]?0[xX]/;t.exports=8!==e(o+"08")||22!==e(o+"0x16")?function(t,n){var r=i(String(t),3);return e(r,n>>>0||(u.test(r)?16:10))}:e},function(t,n){t.exports=Object.is||function(t,n){return t===n?0!==t||1/t==1/n:t!=t&&n!=n}},function(t,n,r){var e=r(16),i=r(149),o=r(46);t.exports=function(t,n,r,u){var c=String(o(t)),f=c.length,a=void 0===r?" ":String(r),s=e(n);if(s<=f||""==a)return c;var l=s-f,h=i.call(a,Math.ceil(l/a.length));return h.length>l&&(h=h.slice(0,l)),u?h+c:c+h}},function(t,n,r){n.f=r(7)},function(t,n,r){"use strict";var e=r(164);t.exports=r(118)("Map",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{get:function(t){var n=e.getEntry(this,t);return n&&n.v},set:function(t,n){return e.def(this,0===t?0:t,n)}},e,!0)},function(t,n,r){r(10)&&"g"!=/./g.flags&&r(11).f(RegExp.prototype,"flags",{configurable:!0,get:r(120)})},function(t,n,r){"use strict";var e=r(164);t.exports=r(118)("Set",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{add:function(t){return e.def(this,t=0===t?0:t,t)}},e)},function(t,n,r){"use strict";var e,i=r(48)(0),o=r(28),u=r(65),c=r(172),f=r(166),a=r(6),s=u.getWeak,l=Object.isExtensible,h=f.ufstore,v={},p=function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},d={get:function(t){if(a(t)){var n=s(t);return!0===n?h(this).get(t):n?n[this._i]:void 0}},set:function(t,n){return f.def(this,t,n)}},y=t.exports=r(118)("WeakMap",p,d,f,!0,!0);7!=(new y).set((Object.freeze||Object)(v),7).get(v)&&(e=f.getConstructor(p),c(e.prototype,d),u.NEED=!0,i(["delete","has","get","set"],function(t){var n=y.prototype,r=n[t];o(n,t,function(n,i){if(a(n)&&!l(n)){this._f||(this._f=new e);var o=this._f[t](n,i);return"set"==t?this:o}return r.call(this,n,i)})}))},,,,function(t,n){"use strict";function r(){var t=document.querySelector("#page-nav");if(t&&!document.querySelector("#page-nav .extend.prev")&&(t.innerHTML='<a class="extend prev disabled" rel="prev">&laquo; Prev</a>'+t.innerHTML),t&&!document.querySelector("#page-nav .extend.next")&&(t.innerHTML=t.innerHTML+'<a class="extend next disabled" rel="next">Next &raquo;</a>'),yiliaConfig&&yiliaConfig.open_in_new){document.querySelectorAll(".article-entry a:not(.article-more-a)").forEach(function(t){var n=t.getAttribute("target");n&&""!==n||t.setAttribute("target","_blank")})}if(yiliaConfig&&yiliaConfig.toc_hide_index){document.querySelectorAll(".toc-number").forEach(function(t){t.style.display="none"})}var n=document.querySelector("#js-aboutme");n&&0!==n.length&&(n.innerHTML=n.innerText)}t.exports={init:r}},function(t,n,r){"use strict";function e(t){return t&&t.__esModule?t:{default:t}}function i(t,n){var r=/\/|index.html/g;return t.replace(r,"")===n.replace(r,"")}function o(){for(var t=document.querySelectorAll(".js-header-menu li a"),n=window.location.pathname,r=0,e=t.length;r<e;r++){var o=t[r];i(n,o.getAttribute("href"))&&(0,h.default)(o,"active")}}function u(t){for(var n=t.offsetLeft,r=t.offsetParent;null!==r;)n+=r.offsetLeft,r=r.offsetParent;return n}function c(t){for(var n=t.offsetTop,r=t.offsetParent;null!==r;)n+=r.offsetTop,r=r.offsetParent;return n}function f(t,n,r,e,i){var o=u(t),f=c(t)-n;if(f-r<=i){var a=t.$newDom;a||(a=t.cloneNode(!0),(0,d.default)(t,a),t.$newDom=a,a.style.position="fixed",a.style.top=(r||f)+"px",a.style.left=o+"px",a.style.zIndex=e||2,a.style.width="100%",a.style.color="#fff"),a.style.visibility="visible",t.style.visibility="hidden"}else{t.style.visibility="visible";var s=t.$newDom;s&&(s.style.visibility="hidden")}}function a(){var t=document.querySelector(".js-overlay"),n=document.querySelector(".js-header-menu");f(t,document.body.scrollTop,-63,2,0),f(n,document.body.scrollTop,1,3,0)}function s(){document.querySelector("#container").addEventListener("scroll",function(t){a()}),window.addEventListener("scroll",function(t){a()}),a()}var l=r(156),h=e(l),v=r(157),p=(e(v),r(382)),d=e(p),y=r(128),g=e(y),b=r(190),m=e(b),x=r(129);(function(){g.default.versions.mobile&&window.screen.width<800&&(o(),s())})(),(0,x.addLoadEvent)(function(){m.default.init()}),t.exports={}},,,,function(t,n,r){(function(t){"use strict";function n(t,n,r){t[n]||Object[e](t,n,{writable:!0,configurable:!0,value:r})}if(r(381),r(391),r(198),t._babelPolyfill)throw new Error("only one instance of babel-polyfill is allowed");t._babelPolyfill=!0;var e="defineProperty";n(String.prototype,"padLeft","".padStart),n(String.prototype,"padRight","".padEnd),"pop,reverse,shift,keys,values,entries,indexOf,every,some,forEach,map,filter,find,findIndex,includes,join,slice,concat,push,splice,unshift,sort,lastIndexOf,reduce,reduceRight,copyWithin,fill".split(",").forEach(function(t){[][t]&&n(Array,t,Function.call.bind([][t]))})}).call(n,function(){return this}())},,,function(t,n,r){r(210),t.exports=r(52).RegExp.escape},,,,function(t,n,r){var e=r(6),i=r(138),o=r(7)("species");t.exports=function(t){var n;return i(t)&&(n=t.constructor,"function"!=typeof n||n!==Array&&!i(n.prototype)||(n=void 0),e(n)&&null===(n=n[o])&&(n=void 0)),void 0===n?Array:n}},function(t,n,r){var e=r(202);t.exports=function(t,n){return new(e(t))(n)}},function(t,n,r){"use strict";var e=r(2),i=r(50),o="number";t.exports=function(t){if("string"!==t&&t!==o&&"default"!==t)throw TypeError("Incorrect hint");return i(e(this),t!=o)}},function(t,n,r){var e=r(72),i=r(125),o=r(116);t.exports=function(t){var n=e(t),r=i.f;if(r)for(var u,c=r(t),f=o.f,a=0;c.length>a;)f.call(t,u=c[a++])&&n.push(u);return n}},function(t,n,r){var e=r(72),i=r(30);t.exports=function(t,n){for(var r,o=i(t),u=e(o),c=u.length,f=0;c>f;)if(o[r=u[f++]]===n)return r}},function(t,n,r){"use strict";var e=r(208),i=r(121),o=r(26);t.exports=function(){for(var t=o(this),n=arguments.length,r=Array(n),u=0,c=e._,f=!1;n>u;)(r[u]=arguments[u++])===c&&(f=!0);return function(){var e,o=this,u=arguments.length,a=0,s=0;if(!f&&!u)return i(t,r,o);if(e=r.slice(),f)for(;n>a;a++)e[a]===c&&(e[a]=arguments[s++]);for(;u>s;)e.push(arguments[s++]);return i(t,e,o)}}},function(t,n,r){t.exports=r(3)},function(t,n){t.exports=function(t,n){var r=n===Object(n)?function(t){return n[t]}:n;return function(n){return String(n).replace(t,r)}}},function(t,n,r){var e=r(1),i=r(209)(/[\\^$*+?.()|[\]{}]/g,"\\$&");e(e.S,"RegExp",{escape:function(t){return i(t)}})},function(t,n,r){var e=r(1);e(e.P,"Array",{copyWithin:r(160)}),r(78)("copyWithin")},function(t,n,r){"use strict";var e=r(1),i=r(48)(4);e(e.P+e.F*!r(47)([].every,!0),"Array",{every:function(t){return i(this,t,arguments[1])}})},function(t,n,r){var e=r(1);e(e.P,"Array",{fill:r(130)}),r(78)("fill")},function(t,n,r){"use strict";var e=r(1),i=r(48)(2);e(e.P+e.F*!r(47)([].filter,!0),"Array",{filter:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(6),o="findIndex",u=!0;o in[]&&Array(1)[o](function(){u=!1}),e(e.P+e.F*u,"Array",{findIndex:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)(o)},function(t,n,r){"use strict";var e=r(1),i=r(48)(5),o="find",u=!0;o in[]&&Array(1)[o](function(){u=!1}),e(e.P+e.F*u,"Array",{find:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)(o)},function(t,n,r){"use strict";var e=r(1),i=r(48)(0),o=r(47)([].forEach,!0);e(e.P+e.F*!o,"Array",{forEach:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(53),i=r(1),o=r(17),u=r(169),c=r(137),f=r(16),a=r(131),s=r(154);i(i.S+i.F*!r(123)(function(t){Array.from(t)}),"Array",{from:function(t){var n,r,i,l,h=o(t),v="function"==typeof this?this:Array,p=arguments.length,d=p>1?arguments[1]:void 0,y=void 0!==d,g=0,b=s(h);if(y&&(d=e(d,p>2?arguments[2]:void 0,2)),void 0==b||v==Array&&c(b))for(n=f(h.length),r=new v(n);n>g;g++)a(r,g,y?d(h[g],g):h[g]);else for(l=b.call(h),r=new v;!(i=l.next()).done;g++)a(r,g,y?u(l,d,[i.value,g],!0):i.value);return r.length=g,r}})},function(t,n,r){"use strict";var e=r(1),i=r(117)(!1),o=[].indexOf,u=!!o&&1/[1].indexOf(1,-0)<0;e(e.P+e.F*(u||!r(47)(o)),"Array",{indexOf:function(t){return u?o.apply(this,arguments)||0:i(this,t,arguments[1])}})},function(t,n,r){var e=r(1);e(e.S,"Array",{isArray:r(138)})},function(t,n,r){"use strict";var e=r(1),i=r(30),o=[].join;e(e.P+e.F*(r(115)!=Object||!r(47)(o)),"Array",{join:function(t){return o.call(i(this),void 0===t?",":t)}})},function(t,n,r){"use strict";var e=r(1),i=r(30),o=r(67),u=r(16),c=[].lastIndexOf,f=!!c&&1/[1].lastIndexOf(1,-0)<0;e(e.P+e.F*(f||!r(47)(c)),"Array",{lastIndexOf:function(t){if(f)return c.apply(this,arguments)||0;var n=i(this),r=u(n.length),e=r-1;for(arguments.length>1&&(e=Math.min(e,o(arguments[1]))),e<0&&(e=r+e);e>=0;e--)if(e in n&&n[e]===t)return e||0;return-1}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(1);e(e.P+e.F*!r(47)([].map,!0),"Array",{map:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(131);e(e.S+e.F*r(4)(function(){function t(){}return!(Array.of.call(t)instanceof t)}),"Array",{of:function(){for(var t=0,n=arguments.length,r=new("function"==typeof this?this:Array)(n);n>t;)i(r,t,arguments[t++]);return r.length=n,r}})},function(t,n,r){"use strict";var e=r(1),i=r(162);e(e.P+e.F*!r(47)([].reduceRight,!0),"Array",{reduceRight:function(t){return i(this,t,arguments.length,arguments[1],!0)}})},function(t,n,r){"use strict";var e=r(1),i=r(162);e(e.P+e.F*!r(47)([].reduce,!0),"Array",{reduce:function(t){return i(this,t,arguments.length,arguments[1],!1)}})},function(t,n,r){"use strict";var e=r(1),i=r(135),o=r(45),u=r(75),c=r(16),f=[].slice;e(e.P+e.F*r(4)(function(){i&&f.call(i)}),"Array",{slice:function(t,n){var r=c(this.length),e=o(this);if(n=void 0===n?r:n,"Array"==e)return f.call(this,t,n);for(var i=u(t,r),a=u(n,r),s=c(a-i),l=Array(s),h=0;h<s;h++)l[h]="String"==e?this.charAt(i+h):this[i+h];return l}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(3);e(e.P+e.F*!r(47)([].some,!0),"Array",{some:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(26),o=r(17),u=r(4),c=[].sort,f=[1,2,3];e(e.P+e.F*(u(function(){f.sort(void 0)})||!u(function(){f.sort(null)})||!r(47)(c)),"Array",{sort:function(t){return void 0===t?c.call(o(this)):c.call(o(this),i(t))}})},function(t,n,r){r(74)("Array")},function(t,n,r){var e=r(1);e(e.S,"Date",{now:function(){return(new Date).getTime()}})},function(t,n,r){"use strict";var e=r(1),i=r(4),o=Date.prototype.getTime,u=function(t){return t>9?t:"0"+t};e(e.P+e.F*(i(function(){return"0385-07-25T07:06:39.999Z"!=new Date(-5e13-1).toISOString()})||!i(function(){new Date(NaN).toISOString()})),"Date",{toISOString:function(){
if(!isFinite(o.call(this)))throw RangeError("Invalid time value");var t=this,n=t.getUTCFullYear(),r=t.getUTCMilliseconds(),e=n<0?"-":n>9999?"+":"";return e+("00000"+Math.abs(n)).slice(e?-6:-4)+"-"+u(t.getUTCMonth()+1)+"-"+u(t.getUTCDate())+"T"+u(t.getUTCHours())+":"+u(t.getUTCMinutes())+":"+u(t.getUTCSeconds())+"."+(r>99?r:"0"+u(r))+"Z"}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50);e(e.P+e.F*r(4)(function(){return null!==new Date(NaN).toJSON()||1!==Date.prototype.toJSON.call({toISOString:function(){return 1}})}),"Date",{toJSON:function(t){var n=i(this),r=o(n);return"number"!=typeof r||isFinite(r)?n.toISOString():null}})},function(t,n,r){var e=r(7)("toPrimitive"),i=Date.prototype;e in i||r(27)(i,e,r(204))},function(t,n,r){var e=Date.prototype,i="Invalid Date",o="toString",u=e[o],c=e.getTime;new Date(NaN)+""!=i&&r(28)(e,o,function(){var t=c.call(this);return t===t?u.call(this):i})},function(t,n,r){var e=r(1);e(e.P,"Function",{bind:r(163)})},function(t,n,r){"use strict";var e=r(6),i=r(32),o=r(7)("hasInstance"),u=Function.prototype;o in u||r(11).f(u,o,{value:function(t){if("function"!=typeof this||!e(t))return!1;if(!e(this.prototype))return t instanceof this;for(;t=i(t);)if(this.prototype===t)return!0;return!1}})},function(t,n,r){var e=r(11).f,i=r(66),o=r(24),u=Function.prototype,c="name",f=Object.isExtensible||function(){return!0};c in u||r(10)&&e(u,c,{configurable:!0,get:function(){try{var t=this,n=(""+t).match(/^\s*function ([^ (]*)/)[1];return o(t,c)||!f(t)||e(t,c,i(5,n)),n}catch(t){return""}}})},function(t,n,r){var e=r(1),i=r(171),o=Math.sqrt,u=Math.acosh;e(e.S+e.F*!(u&&710==Math.floor(u(Number.MAX_VALUE))&&u(1/0)==1/0),"Math",{acosh:function(t){return(t=+t)<1?NaN:t>94906265.62425156?Math.log(t)+Math.LN2:i(t-1+o(t-1)*o(t+1))}})},function(t,n,r){function e(t){return isFinite(t=+t)&&0!=t?t<0?-e(-t):Math.log(t+Math.sqrt(t*t+1)):t}var i=r(1),o=Math.asinh;i(i.S+i.F*!(o&&1/o(0)>0),"Math",{asinh:e})},function(t,n,r){var e=r(1),i=Math.atanh;e(e.S+e.F*!(i&&1/i(-0)<0),"Math",{atanh:function(t){return 0==(t=+t)?t:Math.log((1+t)/(1-t))/2}})},function(t,n,r){var e=r(1),i=r(142);e(e.S,"Math",{cbrt:function(t){return i(t=+t)*Math.pow(Math.abs(t),1/3)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{clz32:function(t){return(t>>>=0)?31-Math.floor(Math.log(t+.5)*Math.LOG2E):32}})},function(t,n,r){var e=r(1),i=Math.exp;e(e.S,"Math",{cosh:function(t){return(i(t=+t)+i(-t))/2}})},function(t,n,r){var e=r(1),i=r(141);e(e.S+e.F*(i!=Math.expm1),"Math",{expm1:i})},function(t,n,r){var e=r(1),i=r(142),o=Math.pow,u=o(2,-52),c=o(2,-23),f=o(2,127)*(2-c),a=o(2,-126),s=function(t){return t+1/u-1/u};e(e.S,"Math",{fround:function(t){var n,r,e=Math.abs(t),o=i(t);return e<a?o*s(e/a/c)*a*c:(n=(1+c/u)*e,r=n-(n-e),r>f||r!=r?o*(1/0):o*r)}})},function(t,n,r){var e=r(1),i=Math.abs;e(e.S,"Math",{hypot:function(t,n){for(var r,e,o=0,u=0,c=arguments.length,f=0;u<c;)r=i(arguments[u++]),f<r?(e=f/r,o=o*e*e+1,f=r):r>0?(e=r/f,o+=e*e):o+=r;return f===1/0?1/0:f*Math.sqrt(o)}})},function(t,n,r){var e=r(1),i=Math.imul;e(e.S+e.F*r(4)(function(){return-5!=i(4294967295,5)||2!=i.length}),"Math",{imul:function(t,n){var r=65535,e=+t,i=+n,o=r&e,u=r&i;return 0|o*u+((r&e>>>16)*u+o*(r&i>>>16)<<16>>>0)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{log10:function(t){return Math.log(t)/Math.LN10}})},function(t,n,r){var e=r(1);e(e.S,"Math",{log1p:r(171)})},function(t,n,r){var e=r(1);e(e.S,"Math",{log2:function(t){return Math.log(t)/Math.LN2}})},function(t,n,r){var e=r(1);e(e.S,"Math",{sign:r(142)})},function(t,n,r){var e=r(1),i=r(141),o=Math.exp;e(e.S+e.F*r(4)(function(){return-2e-17!=!Math.sinh(-2e-17)}),"Math",{sinh:function(t){return Math.abs(t=+t)<1?(i(t)-i(-t))/2:(o(t-1)-o(-t-1))*(Math.E/2)}})},function(t,n,r){var e=r(1),i=r(141),o=Math.exp;e(e.S,"Math",{tanh:function(t){var n=i(t=+t),r=i(-t);return n==1/0?1:r==1/0?-1:(n-r)/(o(t)+o(-t))}})},function(t,n,r){var e=r(1);e(e.S,"Math",{trunc:function(t){return(t>0?Math.floor:Math.ceil)(t)}})},function(t,n,r){"use strict";var e=r(3),i=r(24),o=r(45),u=r(136),c=r(50),f=r(4),a=r(71).f,s=r(31).f,l=r(11).f,h=r(82).trim,v="Number",p=e[v],d=p,y=p.prototype,g=o(r(70)(y))==v,b="trim"in String.prototype,m=function(t){var n=c(t,!1);if("string"==typeof n&&n.length>2){n=b?n.trim():h(n,3);var r,e,i,o=n.charCodeAt(0);if(43===o||45===o){if(88===(r=n.charCodeAt(2))||120===r)return NaN}else if(48===o){switch(n.charCodeAt(1)){case 66:case 98:e=2,i=49;break;case 79:case 111:e=8,i=55;break;default:return+n}for(var u,f=n.slice(2),a=0,s=f.length;a<s;a++)if((u=f.charCodeAt(a))<48||u>i)return NaN;return parseInt(f,e)}}return+n};if(!p(" 0o1")||!p("0b1")||p("+0x1")){p=function(t){var n=arguments.length<1?0:t,r=this;return r instanceof p&&(g?f(function(){y.valueOf.call(r)}):o(r)!=v)?u(new d(m(n)),r,p):m(n)};for(var x,w=r(10)?a(d):"MAX_VALUE,MIN_VALUE,NaN,NEGATIVE_INFINITY,POSITIVE_INFINITY,EPSILON,isFinite,isInteger,isNaN,isSafeInteger,MAX_SAFE_INTEGER,MIN_SAFE_INTEGER,parseFloat,parseInt,isInteger".split(","),S=0;w.length>S;S++)i(d,x=w[S])&&!i(p,x)&&l(p,x,s(d,x));p.prototype=y,y.constructor=p,r(28)(e,v,p)}},function(t,n,r){var e=r(1);e(e.S,"Number",{EPSILON:Math.pow(2,-52)})},function(t,n,r){var e=r(1),i=r(3).isFinite;e(e.S,"Number",{isFinite:function(t){return"number"==typeof t&&i(t)}})},function(t,n,r){var e=r(1);e(e.S,"Number",{isInteger:r(168)})},function(t,n,r){var e=r(1);e(e.S,"Number",{isNaN:function(t){return t!=t}})},function(t,n,r){var e=r(1),i=r(168),o=Math.abs;e(e.S,"Number",{isSafeInteger:function(t){return i(t)&&o(t)<=9007199254740991}})},function(t,n,r){var e=r(1);e(e.S,"Number",{MAX_SAFE_INTEGER:9007199254740991})},function(t,n,r){var e=r(1);e(e.S,"Number",{MIN_SAFE_INTEGER:-9007199254740991})},function(t,n,r){var e=r(1),i=r(178);e(e.S+e.F*(Number.parseFloat!=i),"Number",{parseFloat:i})},function(t,n,r){var e=r(1),i=r(179);e(e.S+e.F*(Number.parseInt!=i),"Number",{parseInt:i})},function(t,n,r){"use strict";var e=r(1),i=r(67),o=r(159),u=r(149),c=1..toFixed,f=Math.floor,a=[0,0,0,0,0,0],s="Number.toFixed: incorrect invocation!",l="0",h=function(t,n){for(var r=-1,e=n;++r<6;)e+=t*a[r],a[r]=e%1e7,e=f(e/1e7)},v=function(t){for(var n=6,r=0;--n>=0;)r+=a[n],a[n]=f(r/t),r=r%t*1e7},p=function(){for(var t=6,n="";--t>=0;)if(""!==n||0===t||0!==a[t]){var r=String(a[t]);n=""===n?r:n+u.call(l,7-r.length)+r}return n},d=function(t,n,r){return 0===n?r:n%2==1?d(t,n-1,r*t):d(t*t,n/2,r)},y=function(t){for(var n=0,r=t;r>=4096;)n+=12,r/=4096;for(;r>=2;)n+=1,r/=2;return n};e(e.P+e.F*(!!c&&("0.000"!==8e-5.toFixed(3)||"1"!==.9.toFixed(0)||"1.25"!==1.255.toFixed(2)||"1000000000000000128"!==(0xde0b6b3a7640080).toFixed(0))||!r(4)(function(){c.call({})})),"Number",{toFixed:function(t){var n,r,e,c,f=o(this,s),a=i(t),g="",b=l;if(a<0||a>20)throw RangeError(s);if(f!=f)return"NaN";if(f<=-1e21||f>=1e21)return String(f);if(f<0&&(g="-",f=-f),f>1e-21)if(n=y(f*d(2,69,1))-69,r=n<0?f*d(2,-n,1):f/d(2,n,1),r*=4503599627370496,(n=52-n)>0){for(h(0,r),e=a;e>=7;)h(1e7,0),e-=7;for(h(d(10,e,1),0),e=n-1;e>=23;)v(1<<23),e-=23;v(1<<e),h(1,1),v(2),b=p()}else h(0,r),h(1<<-n,0),b=p()+u.call(l,a);return a>0?(c=b.length,b=g+(c<=a?"0."+u.call(l,a-c)+b:b.slice(0,c-a)+"."+b.slice(c-a))):b=g+b,b}})},function(t,n,r){"use strict";var e=r(1),i=r(4),o=r(159),u=1..toPrecision;e(e.P+e.F*(i(function(){return"1"!==u.call(1,void 0)})||!i(function(){u.call({})})),"Number",{toPrecision:function(t){var n=o(this,"Number#toPrecision: incorrect invocation!");return void 0===t?u.call(n):u.call(n,t)}})},function(t,n,r){var e=r(1);e(e.S+e.F,"Object",{assign:r(172)})},function(t,n,r){var e=r(1);e(e.S,"Object",{create:r(70)})},function(t,n,r){var e=r(1);e(e.S+e.F*!r(10),"Object",{defineProperties:r(173)})},function(t,n,r){var e=r(1);e(e.S+e.F*!r(10),"Object",{defineProperty:r(11).f})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("freeze",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(30),i=r(31).f;r(49)("getOwnPropertyDescriptor",function(){return function(t,n){return i(e(t),n)}})},function(t,n,r){r(49)("getOwnPropertyNames",function(){return r(174).f})},function(t,n,r){var e=r(17),i=r(32);r(49)("getPrototypeOf",function(){return function(t){return i(e(t))}})},function(t,n,r){var e=r(6);r(49)("isExtensible",function(t){return function(n){return!!e(n)&&(!t||t(n))}})},function(t,n,r){var e=r(6);r(49)("isFrozen",function(t){return function(n){return!e(n)||!!t&&t(n)}})},function(t,n,r){var e=r(6);r(49)("isSealed",function(t){return function(n){return!e(n)||!!t&&t(n)}})},function(t,n,r){var e=r(1);e(e.S,"Object",{is:r(180)})},function(t,n,r){var e=r(17),i=r(72);r(49)("keys",function(){return function(t){return i(e(t))}})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("preventExtensions",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("seal",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(1);e(e.S,"Object",{setPrototypeOf:r(144).set})},function(t,n,r){"use strict";var e=r(114),i={};i[r(7)("toStringTag")]="z",i+""!="[object z]"&&r(28)(Object.prototype,"toString",function(){return"[object "+e(this)+"]"},!0)},function(t,n,r){var e=r(1),i=r(178);e(e.G+e.F*(parseFloat!=i),{parseFloat:i})},function(t,n,r){var e=r(1),i=r(179);e(e.G+e.F*(parseInt!=i),{parseInt:i})},function(t,n,r){"use strict";var e,i,o,u=r(69),c=r(3),f=r(53),a=r(114),s=r(1),l=r(6),h=r(26),v=r(68),p=r(79),d=r(146),y=r(151).set,g=r(143)(),b="Promise",m=c.TypeError,x=c.process,w=c[b],x=c.process,S="process"==a(x),_=function(){},O=!!function(){try{var t=w.resolve(1),n=(t.constructor={})[r(7)("species")]=function(t){t(_,_)};return(S||"function"==typeof PromiseRejectionEvent)&&t.then(_)instanceof n}catch(t){}}(),E=function(t,n){return t===n||t===w&&n===o},P=function(t){var n;return!(!l(t)||"function"!=typeof(n=t.then))&&n},j=function(t){return E(w,t)?new F(t):new i(t)},F=i=function(t){var n,r;this.promise=new t(function(t,e){if(void 0!==n||void 0!==r)throw m("Bad Promise constructor");n=t,r=e}),this.resolve=h(n),this.reject=h(r)},M=function(t){try{t()}catch(t){return{error:t}}},A=function(t,n){if(!t._n){t._n=!0;var r=t._c;g(function(){for(var e=t._v,i=1==t._s,o=0;r.length>o;)!function(n){var r,o,u=i?n.ok:n.fail,c=n.resolve,f=n.reject,a=n.domain;try{u?(i||(2==t._h&&I(t),t._h=1),!0===u?r=e:(a&&a.enter(),r=u(e),a&&a.exit()),r===n.promise?f(m("Promise-chain cycle")):(o=P(r))?o.call(r,c,f):c(r)):f(e)}catch(t){f(t)}}(r[o++]);t._c=[],t._n=!1,n&&!t._h&&N(t)})}},N=function(t){y.call(c,function(){var n,r,e,i=t._v;if(T(t)&&(n=M(function(){S?x.emit("unhandledRejection",i,t):(r=c.onunhandledrejection)?r({promise:t,reason:i}):(e=c.console)&&e.error&&e.error("Unhandled promise rejection",i)}),t._h=S||T(t)?2:1),t._a=void 0,n)throw n.error})},T=function(t){if(1==t._h)return!1;for(var n,r=t._a||t._c,e=0;r.length>e;)if(n=r[e++],n.fail||!T(n.promise))return!1;return!0},I=function(t){y.call(c,function(){var n;S?x.emit("rejectionHandled",t):(n=c.onrejectionhandled)&&n({promise:t,reason:t._v})})},k=function(t){var n=this;n._d||(n._d=!0,n=n._w||n,n._v=t,n._s=2,n._a||(n._a=n._c.slice()),A(n,!0))},L=function(t){var n,r=this;if(!r._d){r._d=!0,r=r._w||r;try{if(r===t)throw m("Promise can't be resolved itself");(n=P(t))?g(function(){var e={_w:r,_d:!1};try{n.call(t,f(L,e,1),f(k,e,1))}catch(t){k.call(e,t)}}):(r._v=t,r._s=1,A(r,!1))}catch(t){k.call({_w:r,_d:!1},t)}}};O||(w=function(t){v(this,w,b,"_h"),h(t),e.call(this);try{t(f(L,this,1),f(k,this,1))}catch(t){k.call(this,t)}},e=function(t){this._c=[],this._a=void 0,this._s=0,this._d=!1,this._v=void 0,this._h=0,this._n=!1},e.prototype=r(73)(w.prototype,{then:function(t,n){var r=j(d(this,w));return r.ok="function"!=typeof t||t,r.fail="function"==typeof n&&n,r.domain=S?x.domain:void 0,this._c.push(r),this._a&&this._a.push(r),this._s&&A(this,!1),r.promise},catch:function(t){return this.then(void 0,t)}}),F=function(){var t=new e;this.promise=t,this.resolve=f(L,t,1),this.reject=f(k,t,1)}),s(s.G+s.W+s.F*!O,{Promise:w}),r(81)(w,b),r(74)(b),o=r(52)[b],s(s.S+s.F*!O,b,{reject:function(t){var n=j(this);return(0,n.reject)(t),n.promise}}),s(s.S+s.F*(u||!O),b,{resolve:function(t){if(t instanceof w&&E(t.constructor,this))return t;var n=j(this);return(0,n.resolve)(t),n.promise}}),s(s.S+s.F*!(O&&r(123)(function(t){w.all(t).catch(_)})),b,{all:function(t){var n=this,r=j(n),e=r.resolve,i=r.reject,o=M(function(){var r=[],o=0,u=1;p(t,!1,function(t){var c=o++,f=!1;r.push(void 0),u++,n.resolve(t).then(function(t){f||(f=!0,r[c]=t,--u||e(r))},i)}),--u||e(r)});return o&&i(o.error),r.promise},race:function(t){var n=this,r=j(n),e=r.reject,i=M(function(){p(t,!1,function(t){n.resolve(t).then(r.resolve,e)})});return i&&e(i.error),r.promise}})},function(t,n,r){var e=r(1),i=r(26),o=r(2),u=(r(3).Reflect||{}).apply,c=Function.apply;e(e.S+e.F*!r(4)(function(){u(function(){})}),"Reflect",{apply:function(t,n,r){var e=i(t),f=o(r);return u?u(e,n,f):c.call(e,n,f)}})},function(t,n,r){var e=r(1),i=r(70),o=r(26),u=r(2),c=r(6),f=r(4),a=r(163),s=(r(3).Reflect||{}).construct,l=f(function(){function t(){}return!(s(function(){},[],t)instanceof t)}),h=!f(function(){s(function(){})});e(e.S+e.F*(l||h),"Reflect",{construct:function(t,n){o(t),u(n);var r=arguments.length<3?t:o(arguments[2]);if(h&&!l)return s(t,n,r);if(t==r){switch(n.length){case 0:return new t;case 1:return new t(n[0]);case 2:return new t(n[0],n[1]);case 3:return new t(n[0],n[1],n[2]);case 4:return new t(n[0],n[1],n[2],n[3])}var e=[null];return e.push.apply(e,n),new(a.apply(t,e))}var f=r.prototype,v=i(c(f)?f:Object.prototype),p=Function.apply.call(t,v,n);return c(p)?p:v}})},function(t,n,r){var e=r(11),i=r(1),o=r(2),u=r(50);i(i.S+i.F*r(4)(function(){Reflect.defineProperty(e.f({},1,{value:1}),1,{value:2})}),"Reflect",{defineProperty:function(t,n,r){o(t),n=u(n,!0),o(r);try{return e.f(t,n,r),!0}catch(t){return!1}}})},function(t,n,r){var e=r(1),i=r(31).f,o=r(2);e(e.S,"Reflect",{deleteProperty:function(t,n){var r=i(o(t),n);return!(r&&!r.configurable)&&delete t[n]}})},function(t,n,r){"use strict";var e=r(1),i=r(2),o=function(t){this._t=i(t),this._i=0;var n,r=this._k=[];for(n in t)r.push(n)};r(139)(o,"Object",function(){var t,n=this,r=n._k;do{if(n._i>=r.length)return{value:void 0,done:!0}}while(!((t=r[n._i++])in n._t));return{value:t,done:!1}}),e(e.S,"Reflect",{enumerate:function(t){return new o(t)}})},function(t,n,r){var e=r(31),i=r(1),o=r(2);i(i.S,"Reflect",{getOwnPropertyDescriptor:function(t,n){return e.f(o(t),n)}})},function(t,n,r){var e=r(1),i=r(32),o=r(2);e(e.S,"Reflect",{getPrototypeOf:function(t){return i(o(t))}})},function(t,n,r){function e(t,n){var r,c,s=arguments.length<3?t:arguments[2];return a(t)===s?t[n]:(r=i.f(t,n))?u(r,"value")?r.value:void 0!==r.get?r.get.call(s):void 0:f(c=o(t))?e(c,n,s):void 0}var i=r(31),o=r(32),u=r(24),c=r(1),f=r(6),a=r(2);c(c.S,"Reflect",{get:e})},function(t,n,r){var e=r(1);e(e.S,"Reflect",{has:function(t,n){return n in t}})},function(t,n,r){var e=r(1),i=r(2),o=Object.isExtensible;e(e.S,"Reflect",{isExtensible:function(t){return i(t),!o||o(t)}})},function(t,n,r){var e=r(1);e(e.S,"Reflect",{ownKeys:r(177)})},function(t,n,r){var e=r(1),i=r(2),o=Object.preventExtensions;e(e.S,"Reflect",{preventExtensions:function(t){i(t);try{return o&&o(t),!0}catch(t){return!1}}})},function(t,n,r){var e=r(1),i=r(144);i&&e(e.S,"Reflect",{setPrototypeOf:function(t,n){i.check(t,n);try{return i.set(t,n),!0}catch(t){return!1}}})},function(t,n,r){function e(t,n,r){var f,h,v=arguments.length<4?t:arguments[3],p=o.f(s(t),n);if(!p){if(l(h=u(t)))return e(h,n,r,v);p=a(0)}return c(p,"value")?!(!1===p.writable||!l(v)||(f=o.f(v,n)||a(0),f.value=r,i.f(v,n,f),0)):void 0!==p.set&&(p.set.call(v,r),!0)}var i=r(11),o=r(31),u=r(32),c=r(24),f=r(1),a=r(66),s=r(2),l=r(6);f(f.S,"Reflect",{set:e})},function(t,n,r){var e=r(3),i=r(136),o=r(11).f,u=r(71).f,c=r(122),f=r(120),a=e.RegExp,s=a,l=a.prototype,h=/a/g,v=/a/g,p=new a(h)!==h;if(r(10)&&(!p||r(4)(function(){return v[r(7)("match")]=!1,a(h)!=h||a(v)==v||"/a/i"!=a(h,"i")}))){a=function(t,n){var r=this instanceof a,e=c(t),o=void 0===n;return!r&&e&&t.constructor===a&&o?t:i(p?new s(e&&!o?t.source:t,n):s((e=t instanceof a)?t.source:t,e&&o?f.call(t):n),r?this:l,a)};for(var d=u(s),y=0;d.length>y;)!function(t){t in a||o(a,t,{configurable:!0,get:function(){return s[t]},set:function(n){s[t]=n}})}(d[y++]);l.constructor=a,a.prototype=l,r(28)(e,"RegExp",a)}r(74)("RegExp")},function(t,n,r){r(119)("match",1,function(t,n,r){return[function(r){"use strict";var e=t(this),i=void 0==r?void 0:r[n];return void 0!==i?i.call(r,e):new RegExp(r)[n](String(e))},r]})},function(t,n,r){r(119)("replace",2,function(t,n,r){return[function(e,i){"use strict";var o=t(this),u=void 0==e?void 0:e[n];return void 0!==u?u.call(e,o,i):r.call(String(o),e,i)},r]})},function(t,n,r){r(119)("search",1,function(t,n,r){return[function(r){"use strict";var e=t(this),i=void 0==r?void 0:r[n];return void 0!==i?i.call(r,e):new RegExp(r)[n](String(e))},r]})},function(t,n,r){r(119)("split",2,function(t,n,e){"use strict";var i=r(122),o=e,u=[].push,c="split",f="length",a="lastIndex";if("c"=="abbc"[c](/(b)*/)[1]||4!="test"[c](/(?:)/,-1)[f]||2!="ab"[c](/(?:ab)*/)[f]||4!="."[c](/(.?)(.?)/)[f]||"."[c](/()()/)[f]>1||""[c](/.?/)[f]){var s=void 0===/()??/.exec("")[1];e=function(t,n){var r=String(this);if(void 0===t&&0===n)return[];if(!i(t))return o.call(r,t,n);var e,c,l,h,v,p=[],d=(t.ignoreCase?"i":"")+(t.multiline?"m":"")+(t.unicode?"u":"")+(t.sticky?"y":""),y=0,g=void 0===n?4294967295:n>>>0,b=new RegExp(t.source,d+"g");for(s||(e=new RegExp("^"+b.source+"$(?!\\s)",d));(c=b.exec(r))&&!((l=c.index+c[0][f])>y&&(p.push(r.slice(y,c.index)),!s&&c[f]>1&&c[0].replace(e,function(){for(v=1;v<arguments[f]-2;v++)void 0===arguments[v]&&(c[v]=void 0)}),c[f]>1&&c.index<r[f]&&u.apply(p,c.slice(1)),h=c[0][f],y=l,p[f]>=g));)b[a]===c.index&&b[a]++;return y===r[f]?!h&&b.test("")||p.push(""):p.push(r.slice(y)),p[f]>g?p.slice(0,g):p}}else"0"[c](void 0,0)[f]&&(e=function(t,n){return void 0===t&&0===n?[]:o.call(this,t,n)});return[function(r,i){var o=t(this),u=void 0==r?void 0:r[n];return void 0!==u?u.call(r,o,i):e.call(String(o),r,i)},e]})},function(t,n,r){"use strict";r(184);var e=r(2),i=r(120),o=r(10),u="toString",c=/./[u],f=function(t){r(28)(RegExp.prototype,u,t,!0)};r(4)(function(){return"/a/b"!=c.call({source:"a",flags:"b"})})?f(function(){var t=e(this);return"/".concat(t.source,"/","flags"in t?t.flags:!o&&t instanceof RegExp?i.call(t):void 0)}):c.name!=u&&f(function(){return c.call(this)})},function(t,n,r){"use strict";r(29)("anchor",function(t){return function(n){return t(this,"a","name",n)}})},function(t,n,r){"use strict";r(29)("big",function(t){return function(){return t(this,"big","","")}})},function(t,n,r){"use strict";r(29)("blink",function(t){return function(){return t(this,"blink","","")}})},function(t,n,r){"use strict";r(29)("bold",function(t){return function(){return t(this,"b","","")}})},function(t,n,r){"use strict";var e=r(1),i=r(147)(!1);e(e.P,"String",{codePointAt:function(t){return i(this,t)}})},function(t,n,r){"use strict";var e=r(1),i=r(16),o=r(148),u="endsWith",c=""[u];e(e.P+e.F*r(134)(u),"String",{endsWith:function(t){var n=o(this,t,u),r=arguments.length>1?arguments[1]:void 0,e=i(n.length),f=void 0===r?e:Math.min(i(r),e),a=String(t);return c?c.call(n,a,f):n.slice(f-a.length,f)===a}})},function(t,n,r){"use strict";r(29)("fixed",function(t){return function(){return t(this,"tt","","")}})},function(t,n,r){"use strict";r(29)("fontcolor",function(t){return function(n){return t(this,"font","color",n)}})},function(t,n,r){"use strict";r(29)("fontsize",function(t){return function(n){return t(this,"font","size",n)}})},function(t,n,r){var e=r(1),i=r(75),o=String.fromCharCode,u=String.fromCodePoint;e(e.S+e.F*(!!u&&1!=u.length),"String",{fromCodePoint:function(t){for(var n,r=[],e=arguments.length,u=0;e>u;){if(n=+arguments[u++],i(n,1114111)!==n)throw RangeError(n+" is not a valid code point");r.push(n<65536?o(n):o(55296+((n-=65536)>>10),n%1024+56320))}return r.join("")}})},function(t,n,r){"use strict";var e=r(1),i=r(148),o="includes";e(e.P+e.F*r(134)(o),"String",{includes:function(t){return!!~i(this,t,o).indexOf(t,arguments.length>1?arguments[1]:void 0)}})},function(t,n,r){"use strict";r(29)("italics",function(t){return function(){return t(this,"i","","")}})},function(t,n,r){"use strict";var e=r(147)(!0);r(140)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,n=this._t,r=this._i;return r>=n.length?{value:void 0,done:!0}:(t=e(n,r),this._i+=t.length,{value:t,done:!1})})},function(t,n,r){"use strict";r(29)("link",function(t){return function(n){return t(this,"a","href",n)}})},function(t,n,r){var e=r(1),i=r(30),o=r(16);e(e.S,"String",{raw:function(t){for(var n=i(t.raw),r=o(n.length),e=arguments.length,u=[],c=0;r>c;)u.push(String(n[c++])),c<e&&u.push(String(arguments[c]));return u.join("")}})},function(t,n,r){var e=r(1);e(e.P,"String",{repeat:r(149)})},function(t,n,r){"use strict";r(29)("small",function(t){return function(){return t(this,"small","","")}})},function(t,n,r){"use strict";var e=r(1),i=r(16),o=r(148),u="startsWith",c=""[u];e(e.P+e.F*r(134)(u),"String",{startsWith:function(t){var n=o(this,t,u),r=i(Math.min(arguments.length>1?arguments[1]:void 0,n.length)),e=String(t);return c?c.call(n,e,r):n.slice(r,r+e.length)===e}})},function(t,n,r){"use strict";r(29)("strike",function(t){return function(){return t(this,"strike","","")}})},function(t,n,r){"use strict";r(29)("sub",function(t){return function(){return t(this,"sub","","")}})},function(t,n,r){"use strict";r(29)("sup",function(t){return function(){return t(this,"sup","","")}})},function(t,n,r){"use strict";r(82)("trim",function(t){return function(){return t(this,3)}})},function(t,n,r){"use strict";var e=r(3),i=r(24),o=r(10),u=r(1),c=r(28),f=r(65).KEY,a=r(4),s=r(126),l=r(81),h=r(76),v=r(7),p=r(182),d=r(153),y=r(206),g=r(205),b=r(138),m=r(2),x=r(30),w=r(50),S=r(66),_=r(70),O=r(174),E=r(31),P=r(11),j=r(72),F=E.f,M=P.f,A=O.f,N=e.Symbol,T=e.JSON,I=T&&T.stringify,k="prototype",L=v("_hidden"),R=v("toPrimitive"),C={}.propertyIsEnumerable,D=s("symbol-registry"),U=s("symbols"),W=s("op-symbols"),G=Object[k],B="function"==typeof N,V=e.QObject,z=!V||!V[k]||!V[k].findChild,q=o&&a(function(){return 7!=_(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,n,r){var e=F(G,n);e&&delete G[n],M(t,n,r),e&&t!==G&&M(G,n,e)}:M,K=function(t){var n=U[t]=_(N[k]);return n._k=t,n},J=B&&"symbol"==typeof N.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof N},Y=function(t,n,r){return t===G&&Y(W,n,r),m(t),n=w(n,!0),m(r),i(U,n)?(r.enumerable?(i(t,L)&&t[L][n]&&(t[L][n]=!1),r=_(r,{enumerable:S(0,!1)})):(i(t,L)||M(t,L,S(1,{})),t[L][n]=!0),q(t,n,r)):M(t,n,r)},H=function(t,n){m(t);for(var r,e=g(n=x(n)),i=0,o=e.length;o>i;)Y(t,r=e[i++],n[r]);return t},$=function(t,n){return void 0===n?_(t):H(_(t),n)},X=function(t){var n=C.call(this,t=w(t,!0));return!(this===G&&i(U,t)&&!i(W,t))&&(!(n||!i(this,t)||!i(U,t)||i(this,L)&&this[L][t])||n)},Q=function(t,n){if(t=x(t),n=w(n,!0),t!==G||!i(U,n)||i(W,n)){var r=F(t,n);return!r||!i(U,n)||i(t,L)&&t[L][n]||(r.enumerable=!0),r}},Z=function(t){for(var n,r=A(x(t)),e=[],o=0;r.length>o;)i(U,n=r[o++])||n==L||n==f||e.push(n);return e},tt=function(t){for(var n,r=t===G,e=A(r?W:x(t)),o=[],u=0;e.length>u;)!i(U,n=e[u++])||r&&!i(G,n)||o.push(U[n]);return o};B||(N=function(){if(this instanceof N)throw TypeError("Symbol is not a constructor!");var t=h(arguments.length>0?arguments[0]:void 0),n=function(r){this===G&&n.call(W,r),i(this,L)&&i(this[L],t)&&(this[L][t]=!1),q(this,t,S(1,r))};return o&&z&&q(G,t,{configurable:!0,set:n}),K(t)},c(N[k],"toString",function(){return this._k}),E.f=Q,P.f=Y,r(71).f=O.f=Z,r(116).f=X,r(125).f=tt,o&&!r(69)&&c(G,"propertyIsEnumerable",X,!0),p.f=function(t){return K(v(t))}),u(u.G+u.W+u.F*!B,{Symbol:N});for(var nt="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),rt=0;nt.length>rt;)v(nt[rt++]);for(var nt=j(v.store),rt=0;nt.length>rt;)d(nt[rt++]);u(u.S+u.F*!B,"Symbol",{for:function(t){return i(D,t+="")?D[t]:D[t]=N(t)},keyFor:function(t){if(J(t))return y(D,t);throw TypeError(t+" is not a symbol!")},useSetter:function(){z=!0},useSimple:function(){z=!1}}),u(u.S+u.F*!B,"Object",{create:$,defineProperty:Y,defineProperties:H,getOwnPropertyDescriptor:Q,getOwnPropertyNames:Z,getOwnPropertySymbols:tt}),T&&u(u.S+u.F*(!B||a(function(){var t=N();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!J(t)){for(var n,r,e=[t],i=1;arguments.length>i;)e.push(arguments[i++]);return n=e[1],"function"==typeof n&&(r=n),!r&&b(n)||(n=function(t,n){if(r&&(n=r.call(this,t,n)),!J(n))return n}),e[1]=n,I.apply(T,e)}}}),N[k][R]||r(27)(N[k],R,N[k].valueOf),l(N,"Symbol"),l(Math,"Math",!0),l(e.JSON,"JSON",!0)},function(t,n,r){"use strict";var e=r(1),i=r(127),o=r(152),u=r(2),c=r(75),f=r(16),a=r(6),s=r(3).ArrayBuffer,l=r(146),h=o.ArrayBuffer,v=o.DataView,p=i.ABV&&s.isView,d=h.prototype.slice,y=i.VIEW,g="ArrayBuffer";e(e.G+e.W+e.F*(s!==h),{ArrayBuffer:h}),e(e.S+e.F*!i.CONSTR,g,{isView:function(t){return p&&p(t)||a(t)&&y in t}}),e(e.P+e.U+e.F*r(4)(function(){return!new h(2).slice(1,void 0).byteLength}),g,{slice:function(t,n){if(void 0!==d&&void 0===n)return d.call(u(this),t);for(var r=u(this).byteLength,e=c(t,r),i=c(void 0===n?r:n,r),o=new(l(this,h))(f(i-e)),a=new v(this),s=new v(o),p=0;e<i;)s.setUint8(p++,a.getUint8(e++));return o}}),r(74)(g)},function(t,n,r){var e=r(1);e(e.G+e.W+e.F*!r(127).ABV,{DataView:r(152).DataView})},function(t,n,r){r(55)("Float32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Float64",8,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int16",2,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int8",1,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint16",2,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint8",1,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint8",1,function(t){return function(n,r,e){return t(this,n,r,e)}},!0)},function(t,n,r){"use strict";var e=r(166);r(118)("WeakSet",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{add:function(t){return e.def(this,t,!0)}},e,!1,!0)},function(t,n,r){"use strict";var e=r(1),i=r(117)(!0);e(e.P,"Array",{includes:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)("includes")},function(t,n,r){var e=r(1),i=r(143)(),o=r(3).process,u="process"==r(45)(o);e(e.G,{asap:function(t){var n=u&&o.domain;i(n?n.bind(t):t)}})},function(t,n,r){var e=r(1),i=r(45);e(e.S,"Error",{isError:function(t){return"Error"===i(t)}})},function(t,n,r){var e=r(1);e(e.P+e.R,"Map",{toJSON:r(165)("Map")})},function(t,n,r){var e=r(1);e(e.S,"Math",{iaddh:function(t,n,r,e){var i=t>>>0,o=n>>>0,u=r>>>0;return o+(e>>>0)+((i&u|(i|u)&~(i+u>>>0))>>>31)|0}})},function(t,n,r){var e=r(1);e(e.S,"Math",{imulh:function(t,n){var r=65535,e=+t,i=+n,o=e&r,u=i&r,c=e>>16,f=i>>16,a=(c*u>>>0)+(o*u>>>16);return c*f+(a>>16)+((o*f>>>0)+(a&r)>>16)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{isubh:function(t,n,r,e){var i=t>>>0,o=n>>>0,u=r>>>0;return o-(e>>>0)-((~i&u|~(i^u)&i-u>>>0)>>>31)|0}})},function(t,n,r){var e=r(1);e(e.S,"Math",{umulh:function(t,n){var r=65535,e=+t,i=+n,o=e&r,u=i&r,c=e>>>16,f=i>>>16,a=(c*u>>>0)+(o*u>>>16);return c*f+(a>>>16)+((o*f>>>0)+(a&r)>>>16)}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(26),u=r(11);r(10)&&e(e.P+r(124),"Object",{__defineGetter__:function(t,n){u.f(i(this),t,{get:o(n),enumerable:!0,configurable:!0})}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(26),u=r(11);r(10)&&e(e.P+r(124),"Object",{__defineSetter__:function(t,n){u.f(i(this),t,{set:o(n),enumerable:!0,configurable:!0})}})},function(t,n,r){var e=r(1),i=r(176)(!0);e(e.S,"Object",{entries:function(t){return i(t)}})},function(t,n,r){var e=r(1),i=r(177),o=r(30),u=r(31),c=r(131);e(e.S,"Object",{getOwnPropertyDescriptors:function(t){for(var n,r=o(t),e=u.f,f=i(r),a={},s=0;f.length>s;)c(a,n=f[s++],e(r,n));return a}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50),u=r(32),c=r(31).f;r(10)&&e(e.P+r(124),"Object",{__lookupGetter__:function(t){var n,r=i(this),e=o(t,!0);do{if(n=c(r,e))return n.get}while(r=u(r))}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50),u=r(32),c=r(31).f;r(10)&&e(e.P+r(124),"Object",{__lookupSetter__:function(t){var n,r=i(this),e=o(t,!0);do{if(n=c(r,e))return n.set}while(r=u(r))}})},function(t,n,r){var e=r(1),i=r(176)(!1);e(e.S,"Object",{values:function(t){return i(t)}})},function(t,n,r){"use strict";var e=r(1),i=r(3),o=r(52),u=r(143)(),c=r(7)("observable"),f=r(26),a=r(2),s=r(68),l=r(73),h=r(27),v=r(79),p=v.RETURN,d=function(t){return null==t?void 0:f(t)},y=function(t){var n=t._c;n&&(t._c=void 0,n())},g=function(t){return void 0===t._o},b=function(t){g(t)||(t._o=void 0,y(t))},m=function(t,n){a(t),this._c=void 0,this._o=t,t=new x(this);try{var r=n(t),e=r;null!=r&&("function"==typeof r.unsubscribe?r=function(){e.unsubscribe()}:f(r),this._c=r)}catch(n){return void t.error(n)}g(this)&&y(this)};m.prototype=l({},{unsubscribe:function(){b(this)}});var x=function(t){this._s=t};x.prototype=l({},{next:function(t){var n=this._s;if(!g(n)){var r=n._o;try{var e=d(r.next);if(e)return e.call(r,t)}catch(t){try{b(n)}finally{throw t}}}},error:function(t){var n=this._s;if(g(n))throw t;var r=n._o;n._o=void 0;try{var e=d(r.error);if(!e)throw t;t=e.call(r,t)}catch(t){try{y(n)}finally{throw t}}return y(n),t},complete:function(t){var n=this._s;if(!g(n)){var r=n._o;n._o=void 0;try{var e=d(r.complete);t=e?e.call(r,t):void 0}catch(t){try{y(n)}finally{throw t}}return y(n),t}}});var w=function(t){s(this,w,"Observable","_f")._f=f(t)};l(w.prototype,{subscribe:function(t){return new m(t,this._f)},forEach:function(t){var n=this;return new(o.Promise||i.Promise)(function(r,e){f(t);var i=n.subscribe({next:function(n){try{return t(n)}catch(t){e(t),i.unsubscribe()}},error:e,complete:r})})}}),l(w,{from:function(t){var n="function"==typeof this?this:w,r=d(a(t)[c]);if(r){var e=a(r.call(t));return e.constructor===n?e:new n(function(t){return e.subscribe(t)})}return new n(function(n){var r=!1;return u(function(){if(!r){try{if(v(t,!1,function(t){if(n.next(t),r)return p})===p)return}catch(t){if(r)throw t;return void n.error(t)}n.complete()}}),function(){r=!0}})},of:function(){for(var t=0,n=arguments.length,r=Array(n);t<n;)r[t]=arguments[t++];return new("function"==typeof this?this:w)(function(t){var n=!1;return u(function(){if(!n){for(var e=0;e<r.length;++e)if(t.next(r[e]),n)return;t.complete()}}),function(){n=!0}})}}),h(w.prototype,c,function(){return this}),e(e.G,{Observable:w}),r(74)("Observable")},function(t,n,r){var e=r(54),i=r(2),o=e.key,u=e.set;e.exp({defineMetadata:function(t,n,r,e){u(t,n,i(r),o(e))}})},function(t,n,r){var e=r(54),i=r(2),o=e.key,u=e.map,c=e.store;e.exp({deleteMetadata:function(t,n){var r=arguments.length<3?void 0:o(arguments[2]),e=u(i(n),r,!1);if(void 0===e||!e.delete(t))return!1;if(e.size)return!0;var f=c.get(n);return f.delete(r),!!f.size||c.delete(n)}})},function(t,n,r){var e=r(185),i=r(161),o=r(54),u=r(2),c=r(32),f=o.keys,a=o.key,s=function(t,n){var r=f(t,n),o=c(t);if(null===o)return r;var u=s(o,n);return u.length?r.length?i(new e(r.concat(u))):u:r};o.exp({getMetadataKeys:function(t){return s(u(t),arguments.length<2?void 0:a(arguments[1]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(32),u=e.has,c=e.get,f=e.key,a=function(t,n,r){if(u(t,n,r))return c(t,n,r);var e=o(n);return null!==e?a(t,e,r):void 0};e.exp({getMetadata:function(t,n){return a(t,i(n),arguments.length<3?void 0:f(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.keys,u=e.key;e.exp({getOwnMetadataKeys:function(t){
return o(i(t),arguments.length<2?void 0:u(arguments[1]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.get,u=e.key;e.exp({getOwnMetadata:function(t,n){return o(t,i(n),arguments.length<3?void 0:u(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(32),u=e.has,c=e.key,f=function(t,n,r){if(u(t,n,r))return!0;var e=o(n);return null!==e&&f(t,e,r)};e.exp({hasMetadata:function(t,n){return f(t,i(n),arguments.length<3?void 0:c(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.has,u=e.key;e.exp({hasOwnMetadata:function(t,n){return o(t,i(n),arguments.length<3?void 0:u(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(26),u=e.key,c=e.set;e.exp({metadata:function(t,n){return function(r,e){c(t,n,(void 0!==e?i:o)(r),u(e))}}})},function(t,n,r){var e=r(1);e(e.P+e.R,"Set",{toJSON:r(165)("Set")})},function(t,n,r){"use strict";var e=r(1),i=r(147)(!0);e(e.P,"String",{at:function(t){return i(this,t)}})},function(t,n,r){"use strict";var e=r(1),i=r(46),o=r(16),u=r(122),c=r(120),f=RegExp.prototype,a=function(t,n){this._r=t,this._s=n};r(139)(a,"RegExp String",function(){var t=this._r.exec(this._s);return{value:t,done:null===t}}),e(e.P,"String",{matchAll:function(t){if(i(this),!u(t))throw TypeError(t+" is not a regexp!");var n=String(this),r="flags"in f?String(t.flags):c.call(t),e=new RegExp(t.source,~r.indexOf("g")?r:"g"+r);return e.lastIndex=o(t.lastIndex),new a(e,n)}})},function(t,n,r){"use strict";var e=r(1),i=r(181);e(e.P,"String",{padEnd:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0,!1)}})},function(t,n,r){"use strict";var e=r(1),i=r(181);e(e.P,"String",{padStart:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0,!0)}})},function(t,n,r){"use strict";r(82)("trimLeft",function(t){return function(){return t(this,1)}},"trimStart")},function(t,n,r){"use strict";r(82)("trimRight",function(t){return function(){return t(this,2)}},"trimEnd")},function(t,n,r){r(153)("asyncIterator")},function(t,n,r){r(153)("observable")},function(t,n,r){var e=r(1);e(e.S,"System",{global:r(3)})},function(t,n,r){for(var e=r(155),i=r(28),o=r(3),u=r(27),c=r(80),f=r(7),a=f("iterator"),s=f("toStringTag"),l=c.Array,h=["NodeList","DOMTokenList","MediaList","StyleSheetList","CSSRuleList"],v=0;v<5;v++){var p,d=h[v],y=o[d],g=y&&y.prototype;if(g){g[a]||u(g,a,l),g[s]||u(g,s,d),c[d]=l;for(p in e)g[p]||i(g,p,e[p],!0)}}},function(t,n,r){var e=r(1),i=r(151);e(e.G+e.B,{setImmediate:i.set,clearImmediate:i.clear})},function(t,n,r){var e=r(3),i=r(1),o=r(121),u=r(207),c=e.navigator,f=!!c&&/MSIE .\./.test(c.userAgent),a=function(t){return f?function(n,r){return t(o(u,[].slice.call(arguments,2),"function"==typeof n?n:Function(n)),r)}:t};i(i.G+i.B+i.F*f,{setTimeout:a(e.setTimeout),setInterval:a(e.setInterval)})},function(t,n,r){r(330),r(269),r(271),r(270),r(273),r(275),r(280),r(274),r(272),r(282),r(281),r(277),r(278),r(276),r(268),r(279),r(283),r(284),r(236),r(238),r(237),r(286),r(285),r(256),r(266),r(267),r(257),r(258),r(259),r(260),r(261),r(262),r(263),r(264),r(265),r(239),r(240),r(241),r(242),r(243),r(244),r(245),r(246),r(247),r(248),r(249),r(250),r(251),r(252),r(253),r(254),r(255),r(317),r(322),r(329),r(320),r(312),r(313),r(318),r(323),r(325),r(308),r(309),r(310),r(311),r(314),r(315),r(316),r(319),r(321),r(324),r(326),r(327),r(328),r(231),r(233),r(232),r(235),r(234),r(220),r(218),r(224),r(221),r(227),r(229),r(217),r(223),r(214),r(228),r(212),r(226),r(225),r(219),r(222),r(211),r(213),r(216),r(215),r(230),r(155),r(302),r(307),r(184),r(303),r(304),r(305),r(306),r(287),r(183),r(185),r(186),r(342),r(331),r(332),r(337),r(340),r(341),r(335),r(338),r(336),r(339),r(333),r(334),r(288),r(289),r(290),r(291),r(292),r(295),r(293),r(294),r(296),r(297),r(298),r(299),r(301),r(300),r(343),r(369),r(372),r(371),r(373),r(374),r(370),r(375),r(376),r(354),r(357),r(353),r(351),r(352),r(355),r(356),r(346),r(368),r(377),r(345),r(347),r(349),r(348),r(350),r(359),r(360),r(362),r(361),r(364),r(363),r(365),r(366),r(367),r(344),r(358),r(380),r(379),r(378),t.exports=r(52)},function(t,n){function r(t,n){if("string"==typeof n)return t.insertAdjacentHTML("afterend",n);var r=t.nextSibling;return r?t.parentNode.insertBefore(n,r):t.parentNode.appendChild(n)}t.exports=r},,,,,,,,,function(t,n,r){(function(n,r){!function(n){"use strict";function e(t,n,r,e){var i=n&&n.prototype instanceof o?n:o,u=Object.create(i.prototype),c=new p(e||[]);return u._invoke=s(t,r,c),u}function i(t,n,r){try{return{type:"normal",arg:t.call(n,r)}}catch(t){return{type:"throw",arg:t}}}function o(){}function u(){}function c(){}function f(t){["next","throw","return"].forEach(function(n){t[n]=function(t){return this._invoke(n,t)}})}function a(t){function n(r,e,o,u){var c=i(t[r],t,e);if("throw"!==c.type){var f=c.arg,a=f.value;return a&&"object"==typeof a&&m.call(a,"__await")?Promise.resolve(a.__await).then(function(t){n("next",t,o,u)},function(t){n("throw",t,o,u)}):Promise.resolve(a).then(function(t){f.value=t,o(f)},u)}u(c.arg)}function e(t,r){function e(){return new Promise(function(e,i){n(t,r,e,i)})}return o=o?o.then(e,e):e()}"object"==typeof r&&r.domain&&(n=r.domain.bind(n));var o;this._invoke=e}function s(t,n,r){var e=P;return function(o,u){if(e===F)throw new Error("Generator is already running");if(e===M){if("throw"===o)throw u;return y()}for(r.method=o,r.arg=u;;){var c=r.delegate;if(c){var f=l(c,r);if(f){if(f===A)continue;return f}}if("next"===r.method)r.sent=r._sent=r.arg;else if("throw"===r.method){if(e===P)throw e=M,r.arg;r.dispatchException(r.arg)}else"return"===r.method&&r.abrupt("return",r.arg);e=F;var a=i(t,n,r);if("normal"===a.type){if(e=r.done?M:j,a.arg===A)continue;return{value:a.arg,done:r.done}}"throw"===a.type&&(e=M,r.method="throw",r.arg=a.arg)}}}function l(t,n){var r=t.iterator[n.method];if(r===g){if(n.delegate=null,"throw"===n.method){if(t.iterator.return&&(n.method="return",n.arg=g,l(t,n),"throw"===n.method))return A;n.method="throw",n.arg=new TypeError("The iterator does not provide a 'throw' method")}return A}var e=i(r,t.iterator,n.arg);if("throw"===e.type)return n.method="throw",n.arg=e.arg,n.delegate=null,A;var o=e.arg;return o?o.done?(n[t.resultName]=o.value,n.next=t.nextLoc,"return"!==n.method&&(n.method="next",n.arg=g),n.delegate=null,A):o:(n.method="throw",n.arg=new TypeError("iterator result is not an object"),n.delegate=null,A)}function h(t){var n={tryLoc:t[0]};1 in t&&(n.catchLoc=t[1]),2 in t&&(n.finallyLoc=t[2],n.afterLoc=t[3]),this.tryEntries.push(n)}function v(t){var n=t.completion||{};n.type="normal",delete n.arg,t.completion=n}function p(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(h,this),this.reset(!0)}function d(t){if(t){var n=t[w];if(n)return n.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var r=-1,e=function n(){for(;++r<t.length;)if(m.call(t,r))return n.value=t[r],n.done=!1,n;return n.value=g,n.done=!0,n};return e.next=e}}return{next:y}}function y(){return{value:g,done:!0}}var g,b=Object.prototype,m=b.hasOwnProperty,x="function"==typeof Symbol?Symbol:{},w=x.iterator||"@@iterator",S=x.asyncIterator||"@@asyncIterator",_=x.toStringTag||"@@toStringTag",O="object"==typeof t,E=n.regeneratorRuntime;if(E)return void(O&&(t.exports=E));E=n.regeneratorRuntime=O?t.exports:{},E.wrap=e;var P="suspendedStart",j="suspendedYield",F="executing",M="completed",A={},N={};N[w]=function(){return this};var T=Object.getPrototypeOf,I=T&&T(T(d([])));I&&I!==b&&m.call(I,w)&&(N=I);var k=c.prototype=o.prototype=Object.create(N);u.prototype=k.constructor=c,c.constructor=u,c[_]=u.displayName="GeneratorFunction",E.isGeneratorFunction=function(t){var n="function"==typeof t&&t.constructor;return!!n&&(n===u||"GeneratorFunction"===(n.displayName||n.name))},E.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,c):(t.__proto__=c,_ in t||(t[_]="GeneratorFunction")),t.prototype=Object.create(k),t},E.awrap=function(t){return{__await:t}},f(a.prototype),a.prototype[S]=function(){return this},E.AsyncIterator=a,E.async=function(t,n,r,i){var o=new a(e(t,n,r,i));return E.isGeneratorFunction(n)?o:o.next().then(function(t){return t.done?t.value:o.next()})},f(k),k[_]="Generator",k.toString=function(){return"[object Generator]"},E.keys=function(t){var n=[];for(var r in t)n.push(r);return n.reverse(),function r(){for(;n.length;){var e=n.pop();if(e in t)return r.value=e,r.done=!1,r}return r.done=!0,r}},E.values=d,p.prototype={constructor:p,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=g,this.done=!1,this.delegate=null,this.method="next",this.arg=g,this.tryEntries.forEach(v),!t)for(var n in this)"t"===n.charAt(0)&&m.call(this,n)&&!isNaN(+n.slice(1))&&(this[n]=g)},stop:function(){this.done=!0;var t=this.tryEntries[0],n=t.completion;if("throw"===n.type)throw n.arg;return this.rval},dispatchException:function(t){function n(n,e){return o.type="throw",o.arg=t,r.next=n,e&&(r.method="next",r.arg=g),!!e}if(this.done)throw t;for(var r=this,e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e],o=i.completion;if("root"===i.tryLoc)return n("end");if(i.tryLoc<=this.prev){var u=m.call(i,"catchLoc"),c=m.call(i,"finallyLoc");if(u&&c){if(this.prev<i.catchLoc)return n(i.catchLoc,!0);if(this.prev<i.finallyLoc)return n(i.finallyLoc)}else if(u){if(this.prev<i.catchLoc)return n(i.catchLoc,!0)}else{if(!c)throw new Error("try statement without catch or finally");if(this.prev<i.finallyLoc)return n(i.finallyLoc)}}}},abrupt:function(t,n){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.tryLoc<=this.prev&&m.call(e,"finallyLoc")&&this.prev<e.finallyLoc){var i=e;break}}i&&("break"===t||"continue"===t)&&i.tryLoc<=n&&n<=i.finallyLoc&&(i=null);var o=i?i.completion:{};return o.type=t,o.arg=n,i?(this.method="next",this.next=i.finallyLoc,A):this.complete(o)},complete:function(t,n){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&n&&(this.next=n),A},finish:function(t){for(var n=this.tryEntries.length-1;n>=0;--n){var r=this.tryEntries[n];if(r.finallyLoc===t)return this.complete(r.completion,r.afterLoc),v(r),A}},catch:function(t){for(var n=this.tryEntries.length-1;n>=0;--n){var r=this.tryEntries[n];if(r.tryLoc===t){var e=r.completion;if("throw"===e.type){var i=e.arg;v(r)}return i}}throw new Error("illegal catch attempt")},delegateYield:function(t,n,r){return this.delegate={iterator:d(t),resultName:n,nextLoc:r},"next"===this.method&&(this.arg=g),A}}}("object"==typeof n?n:"object"==typeof window?window:"object"==typeof self?self:this)}).call(n,function(){return this}(),r(158))}])</script><script src="/./main.0cf68a.js"></script><script>!function(){!function(e){var t=document.createElement("script");document.getElementsByTagName("body")[0].appendChild(t),t.setAttribute("src",e)}("/slider.e37972.js")}()</script>



    
<div class="tools-col" q-class="show:isShow,hide:isShow|isFalse" q-on="click:stop(e)">
  <div class="tools-nav header-menu">
    
    
      
      
      
    
      
      
      
    
      
      
      
    
    

    <ul style="width: 70%">
    
    
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'innerArchive')"><a href="javascript:void(0)" q-class="active:innerArchive">所有文章</a></li>
      
        
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'friends')"><a href="javascript:void(0)" q-class="active:friends">友链</a></li>
      
        
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'aboutme')"><a href="javascript:void(0)" q-class="active:aboutme">关于我</a></li>
      
        
    </ul>
  </div>
  <div class="tools-wrap">
    
    	<section class="tools-section tools-section-all" q-show="innerArchive">
        <div class="search-wrap">
          <input class="search-ipt" q-model="search" type="text" placeholder="find something…">
          <i class="icon-search icon" q-show="search|isEmptyStr"></i>
          <i class="icon-close icon" q-show="search|isNotEmptyStr" q-on="click:clearChose(e)"></i>
        </div>
        <div class="widget tagcloud search-tag">
          <p class="search-tag-wording">tag:</p>
          <label class="search-switch">
            <input type="checkbox" q-on="click:toggleTag(e)" q-attr="checked:showTags">
          </label>
          <ul class="article-tag-list" q-show="showTags">
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">etcd</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">笔记</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">kubernetes</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Statefulset</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">k8s组件</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">DaemonSet</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">Pod</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">HPA</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">Deployment</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">RBAC</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">k8s安装</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">prometheus</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">监控</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Python</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">面向对象</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">野诗</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">古诗</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">随笔</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">hexo</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">cpu性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">linux性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">调度器</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">service</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">卷</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">持久卷</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">pv&pvc</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">Configmap</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Secret</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">准入控制</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">Qos</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">CronJob</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">污点容忍</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">亲和力</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">linux概念</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">linux</a>
              </li>
            
            <div class="clearfix"></div>
          </ul>
        </div>
        <ul class="search-ul">
          <p q-show="jsonFail" style="padding: 20px; font-size: 12px;">
            缺失模块。<br/>1、请确保node版本大于6.2<br/>2、在博客根目录（注意不是yilia根目录）执行以下命令：<br/> npm i hexo-generator-json-content --save<br/><br/>
            3、在根目录_config.yml里添加配置：
<pre style="font-size: 12px;" q-show="jsonFail">
  jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true
</pre>
          </p>
          <li class="search-li" q-repeat="items" q-show="isShow">
            <a q-attr="href:path|urlformat" class="search-title"><i class="icon-quo-left icon"></i><span q-text="title"></span></a>
            <p class="search-time">
              <i class="icon-calendar icon"></i>
              <span q-text="date|dateformat"></span>
            </p>
            <p class="search-tag">
              <i class="icon-price-tags icon"></i>
              <span q-repeat="tags" q-on="click:choseTag(e, name)" q-text="name|tagformat"></span>
            </p>
          </li>
        </ul>
    	</section>
    

    
    	<section class="tools-section tools-section-friends" q-show="friends">
  		
        <ul class="search-ul">
          
            <li class="search-li">
              <a href="https://blog.csdn.net/CN_SHzhaoyujie" target="_blank" class="search-title"><i class="icon-quo-left icon"></i>CSDN</a>
            </li>
          
        </ul>
  		
    	</section>
    

    
    	<section class="tools-section tools-section-me" q-show="aboutme">
  	  	
  	  		<div class="aboutme-wrap" id="js-aboutme">吾字宸轩，号少翁，凡此皆自娱也。&lt;br&gt;&lt;br&gt;少时不学，尝与玩伴溺于游戏。以至无成，每日叹之，或谓曰：&#34;今天下之势，若有所为，须一技傍身。观Linux厚积薄发当世之所趋，君假以时日，定有所成，成则青云唾手。&#34;&lt;br&gt;吾欣然，心甚向之，随往学。司职至今，四年有余，唯小成，闲无事久，志所不愿也，岂有安居而忘者，恐他日必悔之。故求良木，货与之，望诚以待而寻良遇，志所明而求伯乐。&lt;br&gt;凡shell、lamp、lnmp、jenkins、git、ansible、zabbix、mysql、redis无有不通。python、k8s、docker、prometheus、kvm、kettle无有不会。联系请加微信：542238646</div>
  	  	
    	</section>
    
  </div>
  
</div>
    <!-- Root element of PhotoSwipe. Must have class pswp. -->
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

    <!-- Background of PhotoSwipe. 
         It's a separate element as animating opacity is faster than rgba(). -->
    <div class="pswp__bg"></div>

    <!-- Slides wrapper with overflow:hidden. -->
    <div class="pswp__scroll-wrap">

        <!-- Container that holds slides. 
            PhotoSwipe keeps only 3 of them in the DOM to save memory.
            Don't modify these 3 pswp__item elements, data is added later on. -->
        <div class="pswp__container">
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
        </div>

        <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
        <div class="pswp__ui pswp__ui--hidden">

            <div class="pswp__top-bar">

                <!--  Controls are self-explanatory. Order can be changed. -->

                <div class="pswp__counter"></div>

                <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>

                <button class="pswp__button pswp__button--share" style="display:none" title="Share"></button>

                <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>

                <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>

                <!-- Preloader demo http://codepen.io/dimsemenov/pen/yyBWoR -->
                <!-- element will get class pswp__preloader--active when preloader is running -->
                <div class="pswp__preloader">
                    <div class="pswp__preloader__icn">
                      <div class="pswp__preloader__cut">
                        <div class="pswp__preloader__donut"></div>
                      </div>
                    </div>
                </div>
            </div>

            <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
                <div class="pswp__share-tooltip"></div> 
            </div>

            <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
            </button>

            <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
            </button>

            <div class="pswp__caption">
                <div class="pswp__caption__center"></div>
            </div>

        </div>

    </div>

</div>
  </div>
<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"debug":false,"model":{"jsonPath":"/live2dw/assets/hijiki.model.json"},"display":{"position":"right","width":150,"height":500},"mobile":{"show":true},"log":false});</script></body>
</html>